<!DOCTYPE html>
<html>
<head>
  <meta charset="utf-8">
  
  <meta name="renderer" content="webkit">
  <meta http-equiv="X-UA-Compatible" content="IE=edge" >
  <link rel="dns-prefetch" href="http://cxsw_zyj.gitee.io">
  <title>Linux_宸轩少翁</title>
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
  <meta property="og:type" content="website">
<meta property="og:title" content="Linux_宸轩少翁">
<meta property="og:url" content="http://cxsw_zyj.gitee.io/index.html">
<meta property="og:site_name" content="Linux_宸轩少翁">
<meta property="og:locale" content="en_US">
<meta property="article:author" content="宸轩少翁">
<meta name="twitter:card" content="summary">
  
    <link rel="alternative" href="/atom.xml" title="Linux_宸轩少翁" type="application/atom+xml">
  
  
    <link rel="icon" href="/assets/my_favicon.ico">
  
  <link rel="stylesheet" type="text/css" href="/./main.0cf68a.css">
  <style type="text/css">
  
    #container.show {
      background: linear-gradient(200deg,#a0cfe4,#e8c37e);
    }
  </style>
  

  

<meta name="generator" content="Hexo 5.2.0"></head>

<body>
  <div id="container" q-class="show:isCtnShow">
    <canvas id="anm-canvas" class="anm-canvas"></canvas>
    <div class="left-col" q-class="show:isShow">
      
<div class="overlay" style="background: #4d4d4d"></div>
<div class="intrude-less" style="width:100%;">
	<header id="header" class="inner">
		<a href="/" class="profilepic">
			<img src="/assets/1.jpg" class="js-avatar">
		</a>
		<hgroup>
		  <h1 class="header-author"><a href="/"></a></h1>
		</hgroup>
		

		<nav class="header-menu">
			<ul>
			
				<li><a href="/">主页</a></li>
	        
				<li><a href="/categories">分类</a></li>
	        
				<li><a href="/categories/%E6%9D%82%E6%96%87%E9%87%8E%E8%AF%97">杂文野诗</a></li>
	        
			</ul>
		</nav>
		<nav class="header-smart-menu">
    		
    			
    			<a q-on="click: openSlider(e, 'innerArchive')" href="javascript:void(0)">所有文章</a>
    			
            
    			
    			<a q-on="click: openSlider(e, 'friends')" href="javascript:void(0)">友链</a>
    			
            
    			
    			<a q-on="click: openSlider(e, 'aboutme')" href="javascript:void(0)">关于我</a>
    			
            
		</nav>
		<nav class="header-nav">
			<div class="social">
				
					<a class="github" target="_blank" href="#" title="github"><i class="icon-github"></i></a>
		        
					<a class="weibo" target="_blank" href="#" title="weibo"><i class="icon-weibo"></i></a>
		        
					<a class="rss" target="_blank" href="#" title="rss"><i class="icon-rss"></i></a>
		        
					<a class="zhihu" target="_blank" href="#" title="zhihu"><i class="icon-zhihu"></i></a>
		        
			</div>
		</nav>

<div style="position:absolute;left:20px;width:85%">
	<iframe frameborder="no" border="0" marginwidth="0" marginheight="0" width=260 height=86 src="//music.163.com/outchain/player?type=2&id=1325711950&auto=0&height=66"></iframe>
	</div>
	</header>		
</div>

    </div>
    <div class="mid-col" q-class="show:isShow,hide:isShow|isFalse">
      
<nav id="mobile-nav">
  	<div class="overlay js-overlay" style="background: #4d4d4d"></div>
	<div class="btnctn js-mobile-btnctn">
  		<div class="slider-trigger list" q-on="click: openSlider(e)"><i class="icon icon-sort"></i></div>
	</div>
	<div class="intrude-less">
		<header id="header" class="inner">
			<div class="profilepic">
				<img src="/assets/1.jpg" class="js-avatar">
			</div>
			<hgroup>
			  <h1 class="header-author js-header-author"></h1>
			</hgroup>
			
			
			
				
			
				
			
				
			
			
			
			<nav class="header-nav">
				<div class="social">
					
						<a class="github" target="_blank" href="#" title="github"><i class="icon-github"></i></a>
			        
						<a class="weibo" target="_blank" href="#" title="weibo"><i class="icon-weibo"></i></a>
			        
						<a class="rss" target="_blank" href="#" title="rss"><i class="icon-rss"></i></a>
			        
						<a class="zhihu" target="_blank" href="#" title="zhihu"><i class="icon-zhihu"></i></a>
			        
				</div>
			</nav>

			<nav class="header-menu js-header-menu">
				<ul style="width: 70%">
				
				
					<li style="width: 33.333333333333336%"><a href="/">主页</a></li>
		        
					<li style="width: 33.333333333333336%"><a href="/categories">分类</a></li>
		        
					<li style="width: 33.333333333333336%"><a href="/categories/%E6%9D%82%E6%96%87%E9%87%8E%E8%AF%97">杂文野诗</a></li>
		        
				</ul>
			</nav>
		</header>				
	</div>
	<div class="mobile-mask" style="display:none" q-show="isShow"></div>
</nav>

      <div id="wrapper" class="body-wrap">
        <div class="menu-l">
          <div class="canvas-wrap">
            <canvas data-colors="#eaeaea" data-sectionHeight="100" data-contentId="js-content" id="myCanvas1" class="anm-canvas"></canvas>
          </div>
          <div id="js-content" class="content-ll">
            
  
    <article id="post-etcd/etcd日常运维：etcdctl" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/etcd/etcd%E6%97%A5%E5%B8%B8%E8%BF%90%E7%BB%B4%EF%BC%9Aetcdctl/">etcd日常运维:etcdctl客户端命令行</a>
    </h1>
  

        
        <a href="/2021/04/28/etcd/etcd%E6%97%A5%E5%B8%B8%E8%BF%90%E7%BB%B4%EF%BC%9Aetcdctl/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h1 id="1-etcdctl-客户端"><a href="#1-etcdctl-客户端" class="headerlink" title="1 etcdctl 客户端"></a>1 etcdctl 客户端</h1><p>etcdctl 是一个命令行客户端，便于我们进行<strong>服务测试或手动修改数据库内容</strong>，我们刚开始熟悉 etcd 功能时可以通过 etdctl 客户端熟悉相关操作。etcdctl 在两个不同的 etcd 版本（v2 和 v3）下的功能和使用方式也完全不同。一般通过如下方式来指定使用 etcd 的版本：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">export ETCDCTL_API&#x3D;2</span><br><span class="line"></span><br><span class="line">export ETCDCTL_API&#x3D;3</span><br></pre></td></tr></table></figure>



<h1 id="2-常用命令介绍"><a href="#2-常用命令介绍" class="headerlink" title="2 常用命令介绍"></a>2 常用命令介绍</h1><p>我们首先来看下 etcdctl 支持哪些命令，通过<code>etcdctl -h</code>命令查看：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl -h</span><br><span class="line"></span><br><span class="line">NAME:</span><br><span class="line"></span><br><span class="line">	etcdctl - A simple command line client for etcd3.</span><br><span class="line"></span><br><span class="line">USAGE:</span><br><span class="line"></span><br><span class="line">	etcdctl [flags]</span><br><span class="line"></span><br><span class="line">VERSION:</span><br><span class="line"></span><br><span class="line">	3.4.4</span><br><span class="line"></span><br><span class="line">API VERSION:</span><br><span class="line"></span><br><span class="line">	3.4</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>etcdctl 支持的命令大体上分为<strong>数据库操作和非数据库操作</strong>两类。其中数据库的操作命令是最常用的命令，我们将在下面具体介绍。其他的命令如用户、角色、授权、认证相关，你可以根据语法自己尝试一下。</p>
<h1 id="3-数据库操作"><a href="#3-数据库操作" class="headerlink" title="3 数据库操作"></a>3 数据库操作</h1><p>数据库操作基本围绕着对键值和目录的 CRUD 操作（即增删改查），及其对应的生命周期管理。我们上手这些操作其实很方便，因为这些操作是符合 REST 风格的一套 API 操作。</p>
<p>etcd 在键的组织上采用了类似文件系统中目录的概念，即<strong>层次化的空间结构</strong>，我们指定的键可以作为键名，如：testkey，实际上，此时键值对放于根目录 / 下面。我们也可以为键的存储<strong>指定目录结构</strong>，如 /cluster/node/key；如果不存在 /cluster/node 目录，则 etcd Server 将会创建相应的目录结构。</p>
<h2 id="3-1-键操作"><a href="#3-1-键操作" class="headerlink" title="3.1 键操作"></a>3.1 键操作</h2><p>键操作包括最常用的增删改查操作，包括 PUT、GET、DELETE 等命令。</p>
<h3 id="3-1-1-PUT设置或更新"><a href="#3-1-1-PUT设置或更新" class="headerlink" title="3.1.1 PUT设置或更新"></a>3.1.1 PUT设置或更新</h3><ul>
<li><strong>PUT 设置或者更新某个键的值</strong>。例如：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl put &#x2F;test&#x2F;foo1 &quot;Hello world&quot;</span><br><span class="line"></span><br><span class="line">$ etcdctl put &#x2F;test&#x2F;foo2 &quot;Hello world2&quot;</span><br><span class="line"></span><br><span class="line">$ etcdctl put &#x2F;test&#x2F;foo3 &quot;Hello world3&quot;</span><br></pre></td></tr></table></figure>

<p>成功写入三对键值，/test/foo1、/test/foo2 和 /test/foo3。</p>
<h3 id="3-1-2-GET获取"><a href="#3-1-2-GET获取" class="headerlink" title="3.1.2 GET获取"></a>3.1.2 GET获取</h3><h4 id="3-1-2-1-获取指定值"><a href="#3-1-2-1-获取指定值" class="headerlink" title="3.1.2.1 获取指定值"></a>3.1.2.1 获取指定值</h4><ul>
<li><strong>GET 获取指定键的值</strong>。例如获取 /testdir/testkey 对应的值：</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl get &#x2F;testdir&#x2F;testkey</span><br><span class="line"></span><br><span class="line">Hello world</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h4 id="3-1-2-2-获取十六进制的值"><a href="#3-1-2-2-获取十六进制的值" class="headerlink" title="3.1.2.2 获取十六进制的值"></a>3.1.2.2 获取十六进制的值</h4><p>获取其对应的十六进制格式值，即以十六进制格式返回：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl get &#x2F;test&#x2F;foo1 --hex</span><br><span class="line"></span><br><span class="line">\x2f\x74\x65\x73\x74\x64\x69\x72\x2f\x74\x65\x73\x74\x6b\x65\x79 #键</span><br><span class="line"></span><br><span class="line">\x48\x65\x6c\x6c\x6f\x20\x77\x6f\x72\x6c\x64 #值</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>加上<code>--print-value-only</code>可以读取对应的值。十六进制在 etcd 中有多处使用，如<strong>租约 ID</strong> 也是十六进制。</p>
<h4 id="3-1-2-3-获取范围内的值"><a href="#3-1-2-3-获取范围内的值" class="headerlink" title="3.1.2.3 获取范围内的值"></a>3.1.2.3 获取范围内的值</h4><p>GET 范围内的值：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl get &#x2F;test&#x2F;foo1 &#x2F;test&#x2F;foo3</span><br><span class="line"></span><br><span class="line">&#x2F;test&#x2F;foo1</span><br><span class="line"></span><br><span class="line">Hello world</span><br><span class="line"></span><br><span class="line">&#x2F;test&#x2F;foo2</span><br><span class="line"></span><br><span class="line">Hello world2</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>可以看到，上述操作获取了大于等于 /test/foo1，且小于 /test/foo3 的键值对。foo3 不在范围之内，因为范围是半开区间 [foo1, foo3)，不包含 foo3。</p>
<h4 id="3-1-2-4-获取某个前缀的所有键值对"><a href="#3-1-2-4-获取某个前缀的所有键值对" class="headerlink" title="3.1.2.4 获取某个前缀的所有键值对"></a>3.1.2.4 获取某个前缀的所有键值对</h4><p>获取某个前缀的所有键值对，通过 –prefix 可以指定前缀：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl get --prefix &#x2F;test&#x2F;foo</span><br><span class="line"></span><br><span class="line">&#x2F;test&#x2F;foo1</span><br><span class="line"></span><br><span class="line">Hello world</span><br><span class="line"></span><br><span class="line">&#x2F;test&#x2F;foo2</span><br><span class="line"></span><br><span class="line">Hello world2</span><br><span class="line"></span><br><span class="line">&#x2F;test&#x2F;foo3</span><br><span class="line"></span><br><span class="line">Hello world3</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>这样就能获取所有以 /test/foo 开头的键值对，当前缀获取的结果过多时，还可以通过 –limit=2 限制获取的数量：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">etcdctl get --prefix --limit&#x3D;2 &#x2F;test&#x2F;foo</span><br></pre></td></tr></table></figure>



<h4 id="3-1-2-5-获取历史版本的值"><a href="#3-1-2-5-获取历史版本的值" class="headerlink" title="3.1.2.5 获取历史版本的值"></a>3.1.2.5 获取历史版本的值</h4><p>读取键过往版本的值，应用可能想读取键的被替代的值。</p>
<p>例如，应用可能想<strong>通过访问键的过往版本回滚到旧的配置</strong>。或者，应用可能想<strong>通过多个请求得到一个覆盖多个键的统一视图</strong>，而这些请求可以通过访问键历史记录而来。因为 etcd 集群上键值存储的每个修改都会增加 etcd 集群的全局修订版本，应用可以通过提供旧有的 etcd 修改版本来读取被替代的键。现有如下键值对：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">foo &#x3D; bar         # revision &#x3D; 2</span><br><span class="line"></span><br><span class="line">foo1 &#x3D; bar2       # revision &#x3D; 3</span><br><span class="line"></span><br><span class="line">foo &#x3D; bar_new     # revision &#x3D; 4</span><br><span class="line"></span><br><span class="line">foo1 &#x3D; bar1_new   # revision &#x3D; 5</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>以下是访问以前版本 key 的示例：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl get --prefix foo # 访问最新版本的 key</span><br><span class="line"></span><br><span class="line">foo</span><br><span class="line"></span><br><span class="line">bar_new</span><br><span class="line"></span><br><span class="line">foo1</span><br><span class="line"></span><br><span class="line">bar1_new</span><br><span class="line"></span><br><span class="line">$ etcdctl get --prefix --rev&#x3D;4 foo # 访问第 4 个版本的 key</span><br><span class="line"></span><br><span class="line">foo</span><br><span class="line"></span><br><span class="line">bar_new</span><br><span class="line"></span><br><span class="line">foo1</span><br><span class="line"></span><br><span class="line">bar1</span><br><span class="line"></span><br><span class="line">$ etcdctl get --prefix --rev&#x3D;3 foo #  访问第 3 个版本的 key</span><br><span class="line"></span><br><span class="line">foo</span><br><span class="line"></span><br><span class="line">bar</span><br><span class="line"></span><br><span class="line">foo1</span><br><span class="line"></span><br><span class="line">bar1</span><br><span class="line"></span><br><span class="line">$ etcdctl get --prefix --rev&#x3D;2 foo #  访问第 3 个版本的 key</span><br><span class="line"></span><br><span class="line">foo</span><br><span class="line"></span><br><span class="line">bar</span><br><span class="line"></span><br><span class="line">$ etcdctl get --prefix --rev&#x3D;1 foo #  访问第 1 个版本的 key</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h4 id="3-1-2-6-获取大于等于指定键的-byte-值的键"><a href="#3-1-2-6-获取大于等于指定键的-byte-值的键" class="headerlink" title="3.1.2.6 获取大于等于指定键的 byte 值的键"></a>3.1.2.6 获取大于等于指定键的 byte 值的键</h4><p>应用可能想读取大于等于指定键的 byte 值的键。假设 etcd 集群已经有如下列键：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">a &#x3D; 123</span><br><span class="line"></span><br><span class="line">b &#x3D; 456</span><br><span class="line"></span><br><span class="line">z &#x3D; 789</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>读取大于等于键 b 的 byte 值的键的命令：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl get --from-key b</span><br><span class="line"></span><br><span class="line">b</span><br><span class="line"></span><br><span class="line">456</span><br><span class="line"></span><br><span class="line">z</span><br><span class="line"></span><br><span class="line">789</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h3 id="3-1-3-DELETE删除"><a href="#3-1-3-DELETE删除" class="headerlink" title="3.1.3 DELETE删除"></a>3.1.3 DELETE删除</h3><ul>
<li><strong>DELETE 键，应用可以从 etcd 集群中删除一个键或者特定范围的键</strong>。</li>
</ul>
<p>假设 etcd 集群已经有下列键：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line">foo &#x3D; bar</span><br><span class="line"></span><br><span class="line">foo1 &#x3D; bar1</span><br><span class="line"></span><br><span class="line">foo3 &#x3D; bar3</span><br><span class="line"></span><br><span class="line">zoo &#x3D; val</span><br><span class="line"></span><br><span class="line">zoo1 &#x3D; val1</span><br><span class="line"></span><br><span class="line">zoo2 &#x3D; val2</span><br><span class="line"></span><br><span class="line">a &#x3D; 123</span><br><span class="line"></span><br><span class="line">b &#x3D; 456</span><br><span class="line"></span><br><span class="line">z &#x3D; 789</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h4 id="3-1-3-1-删除指定键"><a href="#3-1-3-1-删除指定键" class="headerlink" title="3.1.3.1 删除指定键"></a>3.1.3.1 删除指定键</h4><p>删除键 foo 的命令：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl del foo</span><br><span class="line"></span><br><span class="line">1 # 删除了一个键</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h4 id="3-1-3-2-删除范围内的键"><a href="#3-1-3-2-删除范围内的键" class="headerlink" title="3.1.3.2 删除范围内的键"></a>3.1.3.2 删除范围内的键</h4><p>删除从 foo 到 foo9 范围的键的命令：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl del foo foo9</span><br><span class="line"></span><br><span class="line">2 # 删除了两个键</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h4 id="3-1-3-3-删除并返回键值对"><a href="#3-1-3-3-删除并返回键值对" class="headerlink" title="3.1.3.3 删除并返回键值对"></a>3.1.3.3 删除并返回键值对</h4><p>删除键 zoo 并返回被删除的键值对的命令：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl del --prev-kv zoo</span><br><span class="line"></span><br><span class="line">1   # 一个键被删除</span><br><span class="line"></span><br><span class="line">zoo # 被删除的键</span><br><span class="line"></span><br><span class="line">val # 被删除的键的值</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h4 id="3-1-3-4-删除指定前缀的键"><a href="#3-1-3-4-删除指定前缀的键" class="headerlink" title="3.1.3.4 删除指定前缀的键"></a>3.1.3.4 删除指定前缀的键</h4><p>删除前缀为 zoo 的键的命令：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl del --prefix zoo</span><br><span class="line"></span><br><span class="line">2 # 删除了两个键</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h4 id="3-1-3-5-删除大于等于键-b-的-byte-值的键"><a href="#3-1-3-5-删除大于等于键-b-的-byte-值的键" class="headerlink" title="3.1.3.5 删除大于等于键 b 的 byte 值的键"></a>3.1.3.5 删除大于等于键 b 的 byte 值的键</h4><p>删除大于等于键 b 的 byte 值的键的命令：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl del --from-key b</span><br><span class="line"></span><br><span class="line">2 # 删除了两个键</span><br><span class="line"></span><br></pre></td></tr></table></figure>





<h1 id="4-watch-键值对的改动"><a href="#4-watch-键值对的改动" class="headerlink" title="4 watch 键值对的改动"></a>4 watch 键值对的改动</h1><p>etcd 的 watch 功能是一个常用的功能，我们来看看通过 etcdctl 如何实现 watch 指定的键值对。</p>
<h2 id="4-1-监测指定键"><a href="#4-1-监测指定键" class="headerlink" title="4.1 监测指定键"></a>4.1 监测指定键</h2><p>watch 监测一个键值的变化，一旦<strong>键值发生更新，就会输出最新的值并退出</strong>。例如：用户更新 testkey 键值为 Hello watch。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl watch testkey</span><br><span class="line"></span><br><span class="line"># 在另外一个终端: etcdctl put testkey Hello watch</span><br><span class="line"></span><br><span class="line">testkey</span><br><span class="line"></span><br><span class="line">Hello watch</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h2 id="4-2-监测范围内的键"><a href="#4-2-监测范围内的键" class="headerlink" title="4.2 监测范围内的键"></a>4.2 监测范围内的键</h2><p>从 foo to foo9 范围内键的命令：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl watch foo foo9</span><br><span class="line"></span><br><span class="line"># 在另外一个终端: etcdctl put foo bar</span><br><span class="line"></span><br><span class="line">PUT</span><br><span class="line"></span><br><span class="line">foo</span><br><span class="line"></span><br><span class="line">bar</span><br><span class="line"></span><br><span class="line"># 在另外一个终端: etcdctl put foo1 bar1</span><br><span class="line"></span><br><span class="line">PUT</span><br><span class="line"></span><br><span class="line">foo1</span><br><span class="line"></span><br><span class="line">bar1</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h2 id="4-3-以-16-进制格式在键上检查"><a href="#4-3-以-16-进制格式在键上检查" class="headerlink" title="4.3 以 16 进制格式在键上检查"></a>4.3 以 16 进制格式在键上检查</h2><p>以 16 进制格式在键 foo 上进行观察的命令：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl watch foo --hex</span><br><span class="line"></span><br><span class="line"># 在另外一个终端: etcdctl put foo bar</span><br><span class="line"></span><br><span class="line">PUT</span><br><span class="line"></span><br><span class="line">\x66\x6f\x6f          # 键</span><br><span class="line"></span><br><span class="line">\x62\x61\x72          # 值</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h2 id="4-4-监测多个键"><a href="#4-4-监测多个键" class="headerlink" title="4.4 监测多个键"></a>4.4 监测多个键</h2><p>观察多个键 foo 和 zoo 的命令：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl watch -i</span><br><span class="line"></span><br><span class="line">$ watch foo</span><br><span class="line"></span><br><span class="line">$ watch zoo</span><br><span class="line"></span><br><span class="line"># 在另外一个终端: etcdctl put foo bar</span><br><span class="line"></span><br><span class="line">PUT</span><br><span class="line"></span><br><span class="line">foo</span><br><span class="line"></span><br><span class="line">bar</span><br><span class="line"></span><br><span class="line"># 在另外一个终端: etcdctl put zoo val</span><br><span class="line"></span><br><span class="line">PUT</span><br><span class="line"></span><br><span class="line">zoo</span><br><span class="line"></span><br><span class="line">val</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h2 id="4-5-监测指定版本开始的历史改动"><a href="#4-5-监测指定版本开始的历史改动" class="headerlink" title="4.5 监测指定版本开始的历史改动"></a>4.5 监测指定版本开始的历史改动</h2><p>查看 key 的历史改动，应用可能想观察 etcd 中键的历史改动。</p>
<p>例如，应用服务想要获取某个键的所有修改。如果应用客户端一直与 etcd 服务端保持连接，使用 watch 命令就能够实现了。但是当应用或者 etcd 实例出现异常，该键的改动可能发生在出错期间，这样导致了应用客户端没能实时接收这个更新。因此，<strong>应用客户端必须观察键的历史变动</strong>，为了做到这点，应用客户端可以在观察时指定一个历史修订版本。</p>
<p>首先我们需要完成下述序列的操作：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl put foo bar         # revision &#x3D; 2</span><br><span class="line"></span><br><span class="line">OK</span><br><span class="line"></span><br><span class="line">$ etcdctl put foo1 bar1       # revision &#x3D; 3</span><br><span class="line"></span><br><span class="line">OK</span><br><span class="line"></span><br><span class="line">$ etcdctl put foo bar_new     # revision &#x3D; 4</span><br><span class="line"></span><br><span class="line">OK</span><br><span class="line"></span><br><span class="line">$ etcdctl put foo1 bar1_new   # revision &#x3D; 5</span><br><span class="line"></span><br><span class="line">OK</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h3 id="4-5-1-从指定历史版本开始观察"><a href="#4-5-1-从指定历史版本开始观察" class="headerlink" title="4.5.1 从指定历史版本开始观察"></a>4.5.1 从指定历史版本开始观察</h3><p>观察历史改动：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line"># 从修订版本 2 开始观察键 &#96;foo&#96; 的改动</span><br><span class="line"></span><br><span class="line">$ etcdctl watch --rev&#x3D;2 foo</span><br><span class="line"></span><br><span class="line">PUT</span><br><span class="line"></span><br><span class="line">foo</span><br><span class="line"></span><br><span class="line">bar</span><br><span class="line"></span><br><span class="line">PUT</span><br><span class="line"></span><br><span class="line">foo</span><br><span class="line"></span><br><span class="line">bar_new</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h3 id="4-5-2-从最近一次历史修改开始观察"><a href="#4-5-2-从最近一次历史修改开始观察" class="headerlink" title="4.5.2 从最近一次历史修改开始观察"></a>4.5.2 从最近一次历史修改开始观察</h3><p>从上一次历史修改开始观察：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line"># 在键 &#96;foo&#96; 上观察变更并返回被修改的值和上个修订版本的值</span><br><span class="line"></span><br><span class="line">$ etcdctl watch --prev-kv foo</span><br><span class="line"></span><br><span class="line"># 在另外一个终端: etcdctl put foo bar_latest</span><br><span class="line"></span><br><span class="line">PUT</span><br><span class="line"></span><br><span class="line">foo         # 键</span><br><span class="line"></span><br><span class="line">bar_new     # 在修改前键 foo 的上一个值</span><br><span class="line"></span><br><span class="line">foo         # 键</span><br><span class="line"></span><br><span class="line">bar_latest  # 修改后键 foo 的值</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h3 id="4-5-3-压缩修订版本"><a href="#4-5-3-压缩修订版本" class="headerlink" title="4.5.3 压缩修订版本"></a>4.5.3 压缩修订版本</h3><p>参照上述内容，etcd 保存修订版本以便应用客户端可以读取键的历史版本。但是，为了避免积累无限数量的历史数据，需要对历史的修订版本进行压缩。<strong>经过压缩，etcd 删除历史修订版本，释放存储空间，且在压缩修订版本之前的数据将不可访问</strong>。</p>
<p>下述命令实现了压缩修订版本：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl compact 5</span><br><span class="line"></span><br><span class="line">compacted revision 5 #在压缩修订版本之前的任何修订版本都不可访问</span><br><span class="line"></span><br><span class="line">$ etcdctl get --rev&#x3D;4 foo</span><br><span class="line"></span><br><span class="line">&#123;&quot;level&quot;:&quot;warn&quot;,&quot;ts&quot;:&quot;2020-05-04T16:37:38.020+0800&quot;,&quot;caller&quot;:&quot;clientv3&#x2F;retry_interceptor.go:62&quot;,&quot;msg&quot;:&quot;retrying of unary invoker failed&quot;,&quot;target&quot;:&quot;endpoint:&#x2F;&#x2F;client-c0d35565-0584-4c07-bfeb-034773278656&#x2F;127.0.0.1:2379&quot;,&quot;attempt&quot;:0,&quot;error&quot;:&quot;rpc error: code &#x3D; OutOfRange desc &#x3D; etcdserver: mvcc: required revision has been compacted&quot;&#125;</span><br><span class="line"></span><br><span class="line">Error: etcdserver: mvcc: required revision has been compacted</span><br><span class="line"></span><br></pre></td></tr></table></figure>





<h1 id="5-lease（租约）"><a href="#5-lease（租约）" class="headerlink" title="5 lease（租约）"></a>5 lease（租约）</h1><p>lease 意为租约，类似于 Redis 中的 TTL(Time To Live)。etcd 中的键值对可以绑定到租约上，实现<strong>存活周期控制</strong>。在实际应用中，常用来实现服务的心跳，即服务在启动时获取租约，将租约与服务地址绑定，并写入 etcd 服务器，为了保持心跳状态，服务会定时刷新租约。</p>
<h2 id="5-1-授予租约"><a href="#5-1-授予租约" class="headerlink" title="5.1 授予租约"></a>5.1 <strong>授予租约</strong></h2><p>应用客户端可以为 etcd 集群里面的键授予租约。当键被附加到租约时，它的存活时间被绑定到租约的存活时间，而租约的存活时间相应的被 TTL 管理。在授予租约时，每个租约的最小 TTL 值由应用客户端指定。<strong>一旦租约的 TTL 到期，租约就会过期并且所有附带的键都将被删除</strong>。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"># 授予租约，TTL 为 100 秒</span><br><span class="line"></span><br><span class="line">$ etcdctl lease grant 100</span><br><span class="line"></span><br><span class="line">lease 694d71ddacfda227 granted with TTL(10s)</span><br><span class="line"></span><br><span class="line"># 附加键 foo 到租约 694d71ddacfda227</span><br><span class="line"></span><br><span class="line">$ etcdctl put --lease&#x3D;694d71ddacfda227 foo10 bar</span><br><span class="line"></span><br><span class="line">OK</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>在实际的操作中，<strong>建议 TTL 时间设置久一点</strong>，避免来不及操作而出现如下错误：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">&#123;&quot;level&quot;:&quot;warn&quot;,&quot;ts&quot;:&quot;2020-12-04T17:12:27.957+0800&quot;,&quot;caller&quot;:&quot;clientv3&#x2F;retry_interceptor.go:62&quot;,&quot;msg&quot;:&quot;retrying of unary invoker failed&quot;,&quot;target&quot;:&quot;endpoint:&#x2F;&#x2F;client-f87e9b9e-a583-453b-8781-325f2984cef0&#x2F;127.0.0.1:2379&quot;,&quot;attempt&quot;:0,&quot;error&quot;:&quot;rpc error: code &#x3D; NotFound desc &#x3D; etcdserver: requested lease not found&quot;&#125;</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h2 id="5-2-撤销租约"><a href="#5-2-撤销租约" class="headerlink" title="5.2 撤销租约"></a>5.2 <strong>撤销租约</strong></h2><p>应用通过租约 ID 可以撤销租约。撤销租约将删除所有附带的 key。</p>
<p>我们进行下列操作：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl lease revoke 694d71ddacfda227</span><br><span class="line"></span><br><span class="line">lease 694d71ddacfda227 revoked</span><br><span class="line"></span><br><span class="line">$ etcdctl get foo10</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h2 id="5-3-刷新租期"><a href="#5-3-刷新租期" class="headerlink" title="5.3 刷新租期"></a>5.3 <strong>刷新租期</strong></h2><p>应用程序可以通过刷新其 TTL 保持租约存活，因此不会过期。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl lease keep-alive 694d71ddacfda227</span><br><span class="line"></span><br><span class="line">lease 694d71ddacfda227 keepalived with TTL(100)</span><br><span class="line"></span><br><span class="line">lease 694d71ddacfda227 keepalived with TTL(100)</span><br><span class="line"></span><br><span class="line">...</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h2 id="5-4-查询租期"><a href="#5-4-查询租期" class="headerlink" title="5.4 查询租期"></a>5.4 <strong>查询租期</strong></h2><p>应用客户端可以查询租赁信息，检查续订或租赁的状态，是否存在或者是否已过期。应用客户端还可以查询特定租约绑定的 key。</p>
<p>我们进行下述操作：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl lease grant 300</span><br><span class="line"></span><br><span class="line">lease 694d71ddacfda22c granted with TTL(300s)</span><br><span class="line"></span><br><span class="line">$ etcdctl put --lease&#x3D;694d71ddacfda22c foo10 bar</span><br><span class="line"></span><br><span class="line">OK</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>获取有关租赁信息以及哪些 key 绑定了租赁信息：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl lease timetolive 694d71ddacfda22c</span><br><span class="line"></span><br><span class="line">lease 694d71ddacfda22c granted with TTL(300s), remaining(282s)</span><br><span class="line"></span><br><span class="line">$ etcdctl lease timetolive --keys 694d71ddacfda22c</span><br><span class="line"></span><br><span class="line">lease 694d71ddacfda22c granted with TTL(300s), remaining(220s), attached keys([foo10])</span><br><span class="line"></span><br></pre></td></tr></table></figure>






      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color5">etcd</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/etcd//" class="article-tag-list-link color5">etcd</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/etcd/etcd%E6%97%A5%E5%B8%B8%E8%BF%90%E7%BB%B4%EF%BC%9Aetcdctl/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-etcd/etcd日常运维：动态重配置 etcd 集群" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/etcd/etcd%E6%97%A5%E5%B8%B8%E8%BF%90%E7%BB%B4%EF%BC%9A%E5%8A%A8%E6%80%81%E9%87%8D%E9%85%8D%E7%BD%AE%20etcd%20%E9%9B%86%E7%BE%A4/">etcd日常运维:动态重配置etcd集群</a>
    </h1>
  

        
        <a href="/2021/04/28/etcd/etcd%E6%97%A5%E5%B8%B8%E8%BF%90%E7%BB%B4%EF%BC%9A%E5%8A%A8%E6%80%81%E9%87%8D%E9%85%8D%E7%BD%AE%20etcd%20%E9%9B%86%E7%BE%A4/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h1 id="1-集群运行时重配置"><a href="#1-集群运行时重配置" class="headerlink" title="1 集群运行时重配置"></a>1 集群运行时重配置</h1><p>集群运行时重配置的前提条件是<strong>只有在大多数集群成员都在正常运行时，etcd 集群才能处理重配置请求</strong>。</p>
<p>从两个成员的集群中删除一个成员是不安全的，因为两个成员的集群中的大多数也是两个，如果在删除过程中出现故障，集群就可能无法运行，需要从多数故障中重新启动。因此 etcd 官方建议：<strong>生产环境的集群大小始终大于两个节点</strong>。</p>
<h2 id="1-1-使用场景介绍"><a href="#1-1-使用场景介绍" class="headerlink" title="1.1 使用场景介绍"></a>1.1 使用场景介绍</h2><p>集群的动态重新配置一般的使用场景，如下图所示：</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/%E7%AC%94%E8%AE%B0/image-20210423172047962.png" alt="image-20210423172047962"></p>
<p>如上所述场景中的大多数，都会涉及添加或移除成员。这些操作一般都会使用到 etcd 自带的 etcdctl 命令行工具，命令如下所示：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">member add          已有集群中增加成员</span><br><span class="line"></span><br><span class="line">member remove       移除已有集群中的成员</span><br><span class="line"></span><br><span class="line">member update       更新集群中的成员</span><br><span class="line"></span><br><span class="line">member list         集群成员列表</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p><strong>除了使用 etcdctl 修改成员，还可以使用 etcd v3 gRPC members API</strong>。</p>
<h2 id="1-2-更新成员"><a href="#1-2-更新成员" class="headerlink" title="1.2 更新成员"></a>1.2 更新成员</h2><p>首先我们来看更新成员的实践。<strong>更新成员有两种情况：client URLs</strong> 和 <strong>peer URLs</strong>。</p>
<p>我们回想下这两个配置的功能：</p>
<ul>
<li><p>client URLs 用于客户端的 URL，也就是对外服务的 URL；</p>
</li>
<li><p>peer URLs 用作监听 URL，用于与其他节点通讯。</p>
</li>
</ul>
<p><strong>更新 client URLs</strong></p>
<p>为了更新成员的 client URLs，只需要使用更新后的 client URL 标记（即 –advertise-client-urls）或者环境变量来重启这个成员（ETCD_ADVERTISE_CLIENT_URLS）。重启后的成员将自行发布更新后的 URL，错误更新的 client URL 将不会影响 etcd 集群的健康。</p>
<p><strong>更新 peer URLs</strong></p>
<p>要更新成员的 peer URLs，首先通过成员命令更新它，然后重启成员，因为更新 peer URL 修改了集群范围配置并能影响 etcd 集群的健康。</p>
<p>当我们要更新某个成员的 peer URL，需要找到该目标成员的 ID，使用 etcdctl 列出所有成员：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre></td><td class="code"><pre><span class="line">&#x2F;&#x2F;设置环境变量</span><br><span class="line"></span><br><span class="line">$ ENDPOINTS&#x3D;http:&#x2F;&#x2F;localhost:22379</span><br><span class="line"></span><br><span class="line">&#x2F;&#x2F; 查询所有的集群成员</span><br><span class="line"></span><br><span class="line">$ etcdctl  --endpoints&#x3D;$ENDPOINTS  member list -w table</span><br><span class="line"></span><br><span class="line">+------------------+---------+--------+------------------------+------------------------+------------+</span><br><span class="line"></span><br><span class="line">|        ID        | STATUS  |  NAME  |       PEER ADDRS       |      CLIENT ADDRS      | IS LEARNER |</span><br><span class="line"></span><br><span class="line">+------------------+---------+--------+------------------------+------------------------+------------+</span><br><span class="line"></span><br><span class="line">| 8211f1d0f64f3269 | started | infra1 | http:&#x2F;&#x2F;127.0.0.1:12380 | http:&#x2F;&#x2F;127.0.0.1:12379 |      false |</span><br><span class="line"></span><br><span class="line">| 91bc3c398fb3c146 | started | infra2 | http:&#x2F;&#x2F;127.0.0.1:22380 | http:&#x2F;&#x2F;127.0.0.1:22379 |      false |</span><br><span class="line"></span><br><span class="line">| fd422379fda50e48 | started | infra3 | http:&#x2F;&#x2F;127.0.0.1:32380 | http:&#x2F;&#x2F;127.0.0.1:32379 |      false |</span><br><span class="line"></span><br><span class="line">+------------------+---------+--------+------------------------+------------------------+------------+</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>在这个例子中，我们启动了三个节点的 etcd 集群。更新 8211f1d0f64f3269 成员 ID 并修改它的 peer URLs 值为 <a href="http://127.0.0.1:2380。">http://127.0.0.1:2380。</a></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl   --endpoints&#x3D;http:&#x2F;&#x2F;localhost:12379    member update 8211f1d0f64f3269  --peer-urls&#x3D;http:&#x2F;&#x2F;127.0.0.1:2380</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">Member 8211f1d0f64f3269 updated in cluster ef37ad9dc622a7c4</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>可以看到，集群中 8211f1d0f64f3269 对应的成员信息更新成功。更新之后，集群的成员列表如下所示：</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/%E7%AC%94%E8%AE%B0/image-20210423172244191.png" alt="image-20210423172244191"></p>
<p>随后使用新的配置重启 infra 1，即可完成 etcd 集群成员的 peer URLs 更新。</p>
<h2 id="1-3-删除成员"><a href="#1-3-删除成员" class="headerlink" title="1.3 删除成员"></a>1.3 删除成员</h2><p>基于上面三个节点的集群，假设我们要删除 ID 为 8211f1d0f64f3269 的成员，可使用 remove 命令来执行成员的删除：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl --endpoints&#x3D;$ENDPOINTS member remove 8211f1d0f64f3269</span><br><span class="line"></span><br><span class="line">Member 8211f1d0f64f3269 removed from cluster ef37ad9dc622a7c4</span><br></pre></td></tr></table></figure>

<p>可以看到已经成功执行移除集群中 8211f1d0f64f3269 对应的成员 etcd 1，检查下成员列表进行确认：</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/%E7%AC%94%E8%AE%B0/image-20210423172326764.png" alt="image-20210423172326764"></p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/%E7%AC%94%E8%AE%B0/image-20210423172337399.png" alt="image-20210423172337399"></p>
<p>此时目标成员将会自行关闭服务，并在日志中打印出移除信息：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">13:14:54 etcd1 | &#123;&quot;level&quot;:&quot;warn&quot;,&quot;ts&quot;:&quot;2020-10-18T13:14:54.368+0800&quot;,&quot;caller&quot;:&quot;rafthttp&#x2F;peer_status.go:68&quot;,&quot;msg&quot;:&quot;peer became inactive (message send to peer failed)&quot;,&quot;peer-id&quot;:&quot;fd422379fda50e48&quot;,&quot;error&quot;:&quot;failed to dial fd422379fda50e48 on stream Message (the member has been permanently removed from the cluster)&quot;&#125;</span><br><span class="line"></span><br><span class="line">13:14:54 etcd1 | &#123;&quot;level&quot;:&quot;warn&quot;,&quot;ts&quot;:&quot;2020-10-18T13:14:54.368+0800&quot;,&quot;caller&quot;:&quot;etcdserver&#x2F;server.go:1084&quot;,&quot;msg&quot;:&quot;server error&quot;,&quot;error&quot;:&quot;the member has been permanently removed from the cluster&quot;&#125;</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>这种方式可以安全地移除 leader 和其他成员。<strong>如果是移除 leader 的场景，新 leader 被选举时集群将处于不活动状态（inactive），且持续时间通常由选举超时时间和投票过程决定</strong>。</p>
<h2 id="1-4-添加新成员"><a href="#1-4-添加新成员" class="headerlink" title="1.4 添加新成员"></a>1.4 添加新成员</h2><p>当我们新起节点时，需要加入现有的 etcd 集群中。添加新成员的过程有如下两个步骤：</p>
<ol>
<li><p>通过 HTTP members API 添加新成员到集群，gRPC members API 或者 etcdctl member add 命令；</p>
</li>
<li><p>使用新的集群配置启动新成员，包括更新后的成员列表（现有成员加上新成员）：</p>
</li>
</ol>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl --endpoints&#x3D;http:&#x2F;&#x2F;localhost:22379 member add infra4 --peer-urls&#x3D;http:&#x2F;&#x2F;localhost:2380</span><br><span class="line"></span><br><span class="line">Member 574399926694aee9 added to cluster ef37ad9dc622a7c4</span><br><span class="line"></span><br><span class="line">ETCD_</span><br><span class="line"></span><br><span class="line">ETCD_INITIAL_CLUSTER&#x3D;&quot;infra4&#x3D;http:&#x2F;&#x2F;localhost:2380,infra2&#x3D;http:&#x2F;&#x2F;127.0.0.1:22380,infra3&#x3D;http:&#x2F;&#x2F;127.0.0.1:32380&quot;</span><br><span class="line"></span><br><span class="line">ETCD_INITIAL_ADVERTISE_PEER_URLS&#x3D;&quot;http:&#x2F;&#x2F;localhost:2380&quot;</span><br><span class="line"></span><br><span class="line">ETCD_INITIAL_CLUSTER_STATE&#x3D;&quot;existing&quot;</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>如上的命令使用 etcdctl 指定 name 和 advertised peer URLs 来添加新的成员到集群。我们新增了名为 infra 4 的节点，其启动标志了 –peer-urls=<a href="http://localhost:2380。">http://localhost:2380。</a></p>
<p>通过命令行的输出，可以看到添加成员执行成功。成员 574399926694aee9 添加到集群 ef37ad9dc622a7c4，并在下方输出了集群现有的信息，这些信息很重要。</p>
<p>下面步骤就是基于新的集群配置启动刚刚添加的成员，我们主要是直接使用 etcd 启动的方式：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">etcd --name infra4 --listen-client-urls http:&#x2F;&#x2F;127.0.0.1:2379 --advertise-client-urls http:&#x2F;&#x2F;127.0.0.1:2379 --listen-peer-urls http:&#x2F;&#x2F;127.0.0.1:2380 --initial-advertise-peer-urls http:&#x2F;&#x2F;127.0.0.1:2380 --initial-cluster-token ef37ad9dc622a7c4 --initial-cluster-state existing --initial-cluster &#39;infra4&#x3D;http:&#x2F;&#x2F;127.0.0.1:2380,infra2&#x3D;http:&#x2F;&#x2F;127.0.0.1:22380,infra3&#x3D;http:&#x2F;&#x2F;127.0.0.1:32380&#39; --enable-pprof --logger&#x3D;zap --log-outputs&#x3D;stderr</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>虽然在启动命令中指定了集群的成员、集群的标志、集群状态等信息，但是会出现如下的报错：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">Members:[&amp;&#123;ID:18d3ac4dcf19552b RaftAttributes:&#123;PeerURLs:[http:&#x2F;&#x2F;localhost:2380] IsLearner:false&#125; Attributes:&#123;Name: ClientURLs:[]&#125;&#125; &amp;&#123;ID:91bc3c398fb3c146 RaftAttributes:&#123;PeerURLs:[http:&#x2F;&#x2F;127.0.0.1:22380] IsLearner:false&#125; Attributes:&#123;Name:infra2 ClientURLs:[http:&#x2F;&#x2F;127.0.0.1:22379]&#125;&#125; &amp;&#123;ID:fd422379fda50e48 RaftAttributes:&#123;PeerURLs:[http:&#x2F;&#x2F;127.0.0.1:32380] IsLearner:false&#125; Attributes:&#123;Name:infra3 ClientURLs:[http:&#x2F;&#x2F;127.0.0.1:32379]&#125;&#125;] RemovedMemberIDs:[]&#125;: unmatched member while checking PeerURLs (\&quot;http:&#x2F;&#x2F;127.0.0.1:32380\&quot;(resolved from \&quot;http:&#x2F;&#x2F;127.0.0.1:32380\&quot;) !&#x3D; \&quot;http:&#x2F;&#x2F;127.0.0.1:2380\&quot;(resolved from \&quot;http:&#x2F;&#x2F;127.0.0.1:2380\&quot;))&quot;,&quot;stacktrace&quot;:&quot;go.etcd.io&#x2F;etcd&#x2F;etcdmain.startEtcdOrProxyV2\n\t&#x2F;tmp&#x2F;etcd-release-3.4.5&#x2F;etcd&#x2F;release&#x2F;etcd&#x2F;etcdmain&#x2F;etcd.go:271\ngo.etcd.io&#x2F;etcd&#x2F;etcdmain.Main\n\t&#x2F;tmp&#x2F;etcd-release-3.4.5&#x2F;etcd&#x2F;release&#x2F;etcd&#x2F;etcdmain&#x2F;main.go:46\nmain.main\n\t&#x2F;tmp&#x2F;etcd-release-3.4.5&#x2F;etcd&#x2F;release&#x2F;etcd&#x2F;main.go:28\nruntime.main\n\t&#x2F;usr&#x2F;local&#x2F;go&#x2F;src&#x2F;runtime&#x2F;proc.go:200&quot;&#125;</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>根据报错可以知道，这种方式使得启动的新节点也是集群的方式，peer URLs 不匹配，导致了启动报错。</p>
<p>我们需要知道 etcdctl 添加成员时已经给出关于新成员的集群信息，并打印出成功启动它需要的环境变量。因此使用关联的标记为新的成员启动 etcd 进程：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">$ export ETCD_NAME&#x3D;&quot;infra4&quot;</span><br><span class="line"></span><br><span class="line">$ export ETCD_INITIAL_CLUSTER&#x3D;&quot;infra4&#x3D;http:&#x2F;&#x2F;localhost:2380,infra2&#x3D;http:&#x2F;&#x2F;127.0.0.1:22380,infra3&#x3D;http:&#x2F;&#x2F;127.0.0.1:32380&quot;</span><br><span class="line"></span><br><span class="line">$ export ETCD_INITIAL_CLUSTER_STATE&#x3D;existing</span><br><span class="line"></span><br><span class="line">$ etcd --listen-client-urls http:&#x2F;&#x2F;localhost:2379 --advertise-client-urls http:&#x2F;&#x2F;localhost:2379 --listen-peer-urls http:&#x2F;&#x2F;localhost:2380 --initial-advertise-peer-urls http:&#x2F;&#x2F;localhost:2380</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>如上述的命令执行完成，新成员将作为集群的一部分运行并立即开始同步集群的其他成员。如果添加多个成员，官方推荐的做法是<strong>每次配置单个成员，并在添加更多新成员前验证它正确启动</strong>。</p>
<p>我们此时查看集群的状态如下：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line">$ etcdctl --endpoints&#x3D;http:&#x2F;&#x2F;localhost:22379   member list -w table</span><br><span class="line"></span><br><span class="line">+------------------+---------+--------+------------------------+------------------------+------------+</span><br><span class="line"></span><br><span class="line">|        ID        | STATUS  |  NAME  |       PEER ADDRS       |      CLIENT ADDRS      | IS LEARNER |</span><br><span class="line"></span><br><span class="line">+------------------+---------+--------+------------------------+------------------------+------------+</span><br><span class="line"></span><br><span class="line">| 18d3ac4dcf19552b | started | infra4 | http:&#x2F;&#x2F;localhost:2380  | http:&#x2F;&#x2F;localhost:2379  |      false |</span><br><span class="line"></span><br><span class="line">| 91bc3c398fb3c146 | started | infra2 | http:&#x2F;&#x2F;127.0.0.1:22380 | http:&#x2F;&#x2F;127.0.0.1:22379 |      false |</span><br><span class="line"></span><br><span class="line">| fd422379fda50e48 | started | infra3 | http:&#x2F;&#x2F;127.0.0.1:32380 | http:&#x2F;&#x2F;127.0.0.1:32379 |      false |</span><br><span class="line"></span><br><span class="line">+------------------+---------+--------+------------------------+------------------------+------------+</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/%E7%AC%94%E8%AE%B0/image-20210423172355861.png" alt="image-20210423172355861"></p>
<p>除此之外，<strong>如果添加新成员到一个节点的集群，在新成员启动前集群无法继续工作</strong>，因为它需要两个成员作为 galosh 才能在一致性上达成一致。<strong>这种情况仅会发生</strong>在 etcdctl member add <strong>影响集群和新成员成功建立连接到已有成员的时间内</strong>。</p>
<h2 id="1-5-运行时重配置的设计及注意点"><a href="#1-5-运行时重配置的设计及注意点" class="headerlink" title="1.5 运行时重配置的设计及注意点"></a>1.5 运行时重配置的设计及注意点</h2><p>运行时重配置是分布式系统中难点之一，也很容易出错，我们需要了解运行时重配置命令的设计和注意点。</p>
<h2 id="1-5-1-两阶段配置变更设计"><a href="#1-5-1-两阶段配置变更设计" class="headerlink" title="1.5.1 两阶段配置变更设计"></a>1.5.1 两阶段配置变更设计</h2><p>在 etcd 中，<strong>出于安全考虑，每个 etcd 集群节点进行运行时重配置都必须经历两个阶段：通知集群新配置、加入新成员</strong>。</p>
<p>上面介绍的几种集群操作都是按照这两个步骤进行的。以添加新成员为例，两阶段描述如下。</p>
<ul>
<li>阶段一：通知集群新配置</li>
</ul>
<p>将成员添加到 etcd 集群中，需要通过调用 API 将新成员添加到集群中。当集群同意配置修改时，API 调用返回。</p>
<ul>
<li>阶段二：加入新成员</li>
</ul>
<p>要将新的 etcd 成员加入现有集群，需要指定正确的<code>initial-cluster</code>并将<code>initial-cluster-state</code>设置为 existing。成员启动时，它首先与现有集群通信，并验证当前集群配置是否与<code>initial-cluster</code>中指定的预期配置匹配。当新成员成功启动时，集群已达到预期的配置。</p>
<p>将过程分为两个独立的阶段，运维人员需要了解集群成员身份的变化，这实际上为我们提供了更大的灵活性，也更容易理解这个过程。</p>
<p>我们通过上面的实践可以发现，进行集群运行重配置时，每一阶段都会确认集群成员的数量和状态，当第一阶段没有问题时才会进行下一阶段的操作。这是为了第一阶段的状态不正常时，我们可以及时进行修正，从而避免因为第一阶段的配置问题，导致集群进入无序和混乱的状态。</p>
<h2 id="1-5-2-集群重配置注意点"><a href="#1-5-2-集群重配置注意点" class="headerlink" title="1.5.2 集群重配置注意点"></a>1.5.2 集群重配置注意点</h2><p>我们在前面进行了集群运行时重配置的介绍与实践，但有两点你在重配置时要特别注意。</p>
<ul>
<li><strong>集群永久失去它的大多数成员，需要从旧数据目录启动新集群来恢复之前的状态</strong>。</li>
</ul>
<p>集群永久失去它的大多数成员的情况下，完全有可能从现有集群中强制删除发生故障的成员来完成恢复。但是，etcd 不支持该方法，因为它绕过了不安全的常规共识提交阶段。</p>
<p>如果要删除的成员实际上并没有挂掉或通过同一集群中的不同成员强行删除，etcd 最终会得到具有相同 clusterID 的分散集群。这种方式将会导致后续很难调试和修复。</p>
<ul>
<li><strong>运行时重配置禁止使用公用发现服务</strong></li>
</ul>
<p><a target="_blank" rel="noopener" href="https://etcd.io/docs/v3.4.0/dev-internal/discovery_protocol/">公共发现服务</a>应该仅用于引导集群。成功引导集群后，成员的 IP 地址都是已知的。<strong>若要将新成员加入现有集群，需使用运行时重新配置 API</strong>。</p>
<p>如果依靠公共发现服务会存在一些问题，如公共发现服务自身存在的网络问题、公共发现服务后端是否能够支撑访问负载等。</p>
<p>通过以上的介绍你知道了 etcd 公共发现服务的种种问题。<strong>如果要使用运行时重配置的发现服务，你最好选择构建一个私有服务</strong>。</p>
<h1 id="2-小结"><a href="#2-小结" class="headerlink" title="2 小结"></a>2 小结</h1><p>分布式系统中，运行时集群重配置是一个难点。<strong>运行时重配置会涉及集群的稳定性和可用性，因此需要慎之又慎，尽可能避免运行时集群重配置。</strong>如果你必须重配置 etcd 集群，你需要遵循<strong>两阶段配置变更的思想</strong>，平稳可靠地进行重配置操作。</p>

      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color5">etcd</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/etcd//" class="article-tag-list-link color5">etcd</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/etcd/etcd%E6%97%A5%E5%B8%B8%E8%BF%90%E7%BB%B4%EF%BC%9A%E5%8A%A8%E6%80%81%E9%87%8D%E9%85%8D%E7%BD%AE%20etcd%20%E9%9B%86%E7%BE%A4/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-etcd/etcd日常运维：集群调优" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/etcd/etcd%E6%97%A5%E5%B8%B8%E8%BF%90%E7%BB%B4%EF%BC%9A%E9%9B%86%E7%BE%A4%E8%B0%83%E4%BC%98/">etcd日常运维:集群调优</a>
    </h1>
  

        
        <a href="/2021/04/28/etcd/etcd%E6%97%A5%E5%B8%B8%E8%BF%90%E7%BB%B4%EF%BC%9A%E9%9B%86%E7%BE%A4%E8%B0%83%E4%BC%98/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h1 id="1-etcd-整体分析"><a href="#1-etcd-整体分析" class="headerlink" title="1 etcd 整体分析"></a>1 etcd 整体分析</h1><p>在对 etcd 进行调优之前，我们先来看看 etcd 集群的架构图，如下图所示：</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/%E7%AC%94%E8%AE%B0/image-20210423172425054.png" alt="image-20210423172425054"></p>
<p>上图是一个简化了的 etcd 集群。完整的 etcd 的架构中包括 API 通信层、Raft 算法层、业务逻辑层（包括鉴权、租约等）和 Storage 存储层。</p>
<p>我在图中只标识出了 Raft 层，<strong>Raft 层是实现 etcd 数据一致性的关键</strong>，etcd 节点之间通过 Raft 实现一致性通信。Raft 同步数据需要通过网络，因此网络延迟和网络带宽会影响 etcd 的性能。</p>
<p>还有 Storage 层，Storage 层依赖 BoltDB 作为底层，用以持久化键值对。Storage 层还有 WAL 日志、快照模块。当然，谈起存储势必要提到磁盘 IO 的性能，WAL 日志受到磁盘 IO 写入速度影响，fdatasync 延迟也会影响 etcd 性能。BoltDB Tx 的锁以及 BoltDB 本身的性能也将影响 etcd 的性能。上述这些因素都有可能造成 etcd 的性能损失。</p>
<h1 id="2-推荐的服务器配置"><a href="#2-推荐的服务器配置" class="headerlink" title="2 推荐的服务器配置"></a>2 推荐的服务器配置</h1><p>接下来，我们来看下部署 etcd 集群服务器的配置，这也是我们优化需要首先考虑的内容 。</p>
<p>etcd 在开发或测试的场景下，对硬件要求不高，而且也能运行良好。比如我们在笔记本电脑或低配置服务器上就可使用 etcd 进行开发测试。然而在实际生产环境中运行 etcd 集群时，对于性能等方面的要求就变得很高了，比如 etcd 集群对外提供服务时要求的高可用性和可靠性。因此，匹配的硬件环境是进行生产部署的良好开端。下面我就从 CPU 处理器、内存、磁盘和网络几个方面，具体介绍 etcd 官方推荐的生产环境配置。</p>
<h2 id="2-1-CPU-处理器"><a href="#2-1-CPU-处理器" class="headerlink" title="2.1 CPU 处理器"></a>2.1 <strong>CPU 处理器</strong></h2><p>大部分情况下，etcd 的部署对 CPU 处理器的要求不高。一般的集群只需要双核到四核的 CPU 就能平稳运行。如果 etcd 集群<strong>负载的客户端达到数千个，每秒的请求数可能是成千上万个</strong>，这种情况就需要增加 CPU 的配置，通常需要<strong>八到十六个专用内核</strong>。</p>
<h2 id="2-2-内存大小"><a href="#2-2-内存大小" class="headerlink" title="2.2 内存大小"></a>2.2 <strong>内存大小</strong></h2><p>etcd 对内存的需求同样也不是很高，etcd 服务端内存占用相对较小。当然，即使这样我们也得分配足够的内存给 etcd，通常 8GB 大小的内存就足够了。etcd 服务器会缓存键值数据，其余大部分内存用于跟踪 watch 监视器。因此，<strong>对于具有数千个 watch 监视器或者数百万键值对的大型部署，我们需要相应地将内存扩展到 16GB 以上</strong>。</p>
<h2 id="2-3-磁盘"><a href="#2-3-磁盘" class="headerlink" title="2.3 磁盘"></a>2.3 <strong>磁盘</strong></h2><p>磁盘 IO 速度是影响 etcd 集群性能和稳定性的最关键因素。IO 速度慢的磁盘会增加 etcd 请求的延迟，并有可能影响集群的稳定性。etcd 的一致性共识算法 Raft 依赖元数据，持久存储在日志中，因此<strong>大多数 etcd 集群成员须将请求写入磁盘</strong>。</p>
<p>另外，etcd 还将以增量的方式将检查点写入磁盘中，以便截断该日志。如果这些写入花费的时间太长，心跳可能会超时并触发选举，进而破坏集群的稳定性。通常，可以<strong>使用基准测试工具判断磁盘的速度是否适合 etcd</strong>，为了测量实际的顺序 IOPS，建议使用磁盘基准测试工具，例如 DiskBench 或者 fio。</p>
<p>etcd 对磁盘写入延迟非常敏感，通常需要 7200 RPM 转速的磁盘。对于<strong>负载较重的集群，官方建议使用 SSD 固态硬盘</strong>。etcd 仅需要适度的磁盘带宽，但是当故障成员需要赶上集群时，更大的磁盘带宽可以缩短恢复时间。通常，10MB/s 的带宽可以在 15 秒内恢复 100MB 数据，对于大型集群，建议使用 100MB/s 或更高的速度在 15 秒内恢复 1GB 数据。</p>
<p>在条件允许的情况下，<strong>一般使用 SSD 作为 etcd 的存储</strong>。与机械硬盘相比，SSD 写入延迟较低，能够提高 etcd 的稳定性和可靠性。如果使用机械硬盘，尽量使用转速达到 15,000 RPM 的磁盘。对于机械磁盘和 SSD，使用 RAID 0 也是提高磁盘速度的有效方法。由于 etcd 的一致复制已经获得了高可用性，至少三个集群成员不需要 RAID 的镜像和磁盘阵列。</p>
<h2 id="2-4-网络"><a href="#2-4-网络" class="headerlink" title="2.4 网络"></a>2.4 <strong>网络</strong></h2><p>多个成员的 etcd 集群部署得益于快速可靠的网络。为了使 etcd 既能实现一致性，又能够实现容忍分区性，<strong>需要网络保证低延迟和高带宽</strong>。低延迟使得 etcd 成员可以快速通信，高带宽可以减少恢复故障 etcd 成员的时间，具有分区中断的不可靠网络将导致 etcd 集群的可用性降低。1GbE 对于常见的 etcd 部署就足够了，对于大型 etcd 集群，10GbE 的网络可以缩短平均恢复时间。</p>
<p>我们还可以通过<strong>规避在多个数据中心部署 etcd 成员</strong>的方式来减少网络开销，单个数据中心内部署 etcd 成员可以避免延迟开销，提升 etcd 集群的可用性。</p>
<h1 id="3-etcd-调优"><a href="#3-etcd-调优" class="headerlink" title="3 etcd 调优"></a>3 etcd 调优</h1><p><strong>etcd 启动时的默认设置适用于网络低延迟的场景</strong>，网络延迟较高的场景下，如网络跨域数据中心，心跳间隔和选举超时的设置就需要优化。每一次超时时间应该包含一个请求从发出到响应成功的时间，当然网络慢不仅是延迟导致的，还可能受到 etcd 集群成员的低速磁盘 IO 影响。</p>
<h2 id="3-1-磁盘调优"><a href="#3-1-磁盘调优" class="headerlink" title="3.1 磁盘调优"></a>3.1 <strong>磁盘</strong>调优</h2><p>etcd 集群对磁盘的延迟非常敏感。因为 etcd 需要存储变更日志，多个进程同时操作磁盘可能引起更高的 fsync 延迟。IO 的延迟问题可能引起 etcd 丢失心跳、请求超时或者 Leader 临时丢失，可以通过提高 etcd 进程的磁盘优先级来解决磁盘延迟问题。</p>
<p>在 Linux 系统中，etcd 的磁盘优先级可以通过 Ionic 去配置，我们来看下 Ionice 的命令：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre></td><td class="code"><pre><span class="line">[root@etcd1 ~]# ionice -h</span><br><span class="line"></span><br><span class="line">ionice - sets or gets process io scheduling class and priority.</span><br><span class="line"></span><br><span class="line">Usage:</span><br><span class="line"></span><br><span class="line">  ionice [OPTION] -p PID [PID...]</span><br><span class="line"></span><br><span class="line">  ionice [OPTION] COMMAND</span><br><span class="line"></span><br><span class="line">Options:</span><br><span class="line"></span><br><span class="line">  -c, --class &lt;class&gt;   scheduling class name or number</span><br><span class="line"></span><br><span class="line">                           0: none, 1: realtime, 2: best-effort, 3: idle</span><br><span class="line"></span><br><span class="line">  -n, --classdata &lt;num&gt; scheduling class data</span><br><span class="line"></span><br><span class="line">                           0-7 for realtime and best-effort classes</span><br><span class="line"></span><br><span class="line">  -p, --pid&#x3D;PID         view or modify already running process</span><br><span class="line"></span><br><span class="line">  -t, --ignore          ignore failures</span><br><span class="line"></span><br><span class="line">  -V, --version         output version information and exit</span><br><span class="line"></span><br><span class="line">  -h, --help            display this help and exit</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>根据 Ionice 的提示，我们知道 Ionice 用来获取或设置程序的 IO 调度与优先级。因此，我们可以执行如下的命令：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ sudo ionice -c2 -n0 -p &#96;pgrep etcd&#96;</span><br></pre></td></tr></table></figure>

<p>上述命令指定<code>-c2</code>尽最大努力的调度策略，即操作系统将会尽最大努力设置 etcd 进程为最高优先级。</p>
<h2 id="3-2-网络调优"><a href="#3-2-网络调优" class="headerlink" title="3.2 网络调优"></a>3.2 <strong>网络调优</strong></h2><p>如果 etcd 集群的 Leader 实例拥有大量并发客户端连接，<strong>网络延迟可能会导致 Follower 成员与 Leader 之间通信的请求处理被延迟</strong>。在 Follower 的 Send Buffer 中能看到错误的列表，类似如下的错误：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">dropped MsgProp to 917ad13ee8235c3a since streamMsg&#39;s sending buffer is full</span><br><span class="line"></span><br><span class="line">dropped MsgAppResp to 917ad13ee8235c3a since streamMsg&#39;s sending buffer is full</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>面对这种情况，你可以<strong>通过提高 Leader 的网络优先级来提高 Follower 的请求的响应</strong>。在 Linux 系统中，你可以使用<strong>流量控制机制</strong>来确定对等流量的优先级。流量控制器 TC（Traffic Control）用于 Linux 内核的流量控制，其实现主要是通过在输出端口处建立一个队列来实现流量控制。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">tc qdisc add dev ens192 root handle 1: prio bands 3</span><br><span class="line"></span><br><span class="line">tc filter add dev ens192 parent 1: protocol ip prio 1 u32 match ip sport 2380 0xffff flowid 1:1</span><br><span class="line"></span><br><span class="line">tc filter add dev ens192 parent 1: protocol ip prio 1 u32 match ip dport 2380 0xffff flowid 1:1</span><br><span class="line"></span><br><span class="line">tc filter add dev ens192 parent 1: protocol ip prio 2 u32 match ip sport 2379 0xffff flowid 1:1</span><br><span class="line"></span><br><span class="line">tc filter add dev ens192 parent 1: protocol ip prio 2 u32 match ip dport 2379 0xffff flowid 1:1</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>如上的五条命令中，<code>protocol ip</code>表示该过滤器应该检查报文分组的协议字段。<code>prio 1</code>表示它们对报文处理的优先级，对于不同优先级的过滤器，系统将按照从小到大的优先级排序。其中第一条命令，建立一个优先级队列，并将该队列绑定到网络物理设备 ens192 上，其编号为 1:0。我们可以查看本地网卡的名称，以我的 Centos 7 为例，可以观察到本地的网卡名称为 ens192。</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/%E7%AC%94%E8%AE%B0/image-20210423172611439.png" alt="image-20210423172611439"></p>
<p>接着有四条过滤器的命令，过滤器主要服务于分类。通过上述代码，可以观察到：用于成员间通信的 2380 端口的命令优先级高于 2379 端口。每一个端口有两条命令，分别对应：sport 和 dport。依次执行过滤器，对于相同的优先级，系统将按照命令的先后顺序执行。这几个过滤器还用到了 u32 选择器 (命令中 u32 后面的部分) 来匹配不同的数据流。</p>
<p>第二条和第三条命令，判断的是 dport 和 sport 字段，表示出去或者进来的不同类数据包。如果该字段与<code>Oxffff</code>进行与操作的结果是 2380，则<code>flowid 1:1</code>表示将把该数据流分配给类别 1:1。通过 TC 命令你能够提高 Leader 与 etcd 集群成员之间的网络优先级，使得 etcd 集群处于一个可靠的状态。更加详细的有关 TC 的用法这里我就不再赘述了，你可以参考 <a target="_blank" rel="noopener" href="https://wenku.baidu.com/view/d91bebbbfd0a79563c1e72dd.html">TC 的手册页</a>。</p>
<h2 id="3-3-快照调优"><a href="#3-3-快照调优" class="headerlink" title="3.3 快照调优"></a>3.3 <strong>快照</strong>调优</h2><p>etcd 追加所有键值对的变更到日志中，这些日志每一行记录一个 key 的变更，日志规模在不断增长。当简单使用 etcd 时，这些日志增长不会有问题，但集群规模比较大的时候，问题就会显现，日志就会越来越多且数据量也会变得越来越大。</p>
<p>为了避免大量日志，etcd 会定期生成快照。这些快照通过将当前状态的修改保存到日志，并移除旧的日志，以实现日志的压缩。</p>
<p>创建快照对于 etcd v2 版本来说开销比较大，所以只有当更改记录操作达到一定数量后，才会制作快照。在 etcd 中，默认创建快照的配置是每 10000 次更改才会保存快照，如果 etcd 的内存和磁盘使用率过高，也可以降低这个阈值，命令如下所示：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">$ etcd --snapshot-count&#x3D;5000</span><br><span class="line"></span><br><span class="line">#或者使用环境变量的方式</span><br><span class="line"></span><br><span class="line">$ ETCD_SNAPSHOT_COUNT&#x3D;5000 etcd</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>使用如上两种方式，都可以实现 etcd 实例修改达到 5000 次就会保存快照。</p>
<h2 id="3-4-时间参数调优"><a href="#3-4-时间参数调优" class="headerlink" title="3.4 时间参数调优"></a>3.4 时间参数调优</h2><p>基本的分布式一致性协议依赖于两个单独的时间参数，分别是心跳间隔和选举超时：</p>
<ul>
<li><p>心跳间隔（Heartbeat Interval），该参数通常用来保活，代表 Leader 通知所有的 Follower，它还活着，仍然是 Leader，该参数被设置为节点之间网络往返时间，etcd 默认心跳间隔是 100ms；</p>
</li>
<li><p>选举超时（Election Timeout），它表示 Follower 在多久后还没有收到 Leader 的心跳，它就自己尝试重新发起选举变成 Leader，一般为了避免脑裂发生，这个时间会稍微长一点，etcd 的默认选举超时是 1000ms，当然如果时间太长也会导致数据一致性的问题。</p>
</li>
</ul>
<p>一个 etcd 集群中的所有节点应该设置一样的心跳间隔和选举超时。如果设置不一样可能导致集群不稳定。默认值可以通过命令行参数或环境参数覆盖，如下所示，单位是 ms。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"># 令行参数:</span><br><span class="line"></span><br><span class="line">$ etcd --heartbeat-interval&#x3D;100 --election-timeout&#x3D;500</span><br><span class="line"></span><br><span class="line"># 环境参数:</span><br><span class="line"></span><br><span class="line">$ ETCD_HEARTBEAT_INTERVAL&#x3D;100 ETCD_ELECTION_TIMEOUT&#x3D;500 etcd</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>我们在命令中设置了心跳间隔为 100ms，选举超时为 500ms。对应的环境变量设置在下方，较为方便。</p>
<p>当然，你在实际调整参数时需要做一些权衡，需要考虑网络、服务硬件、负载、集群的规模等因素。<strong>心跳间隔推荐设置为节点之间的最大 RTT，一般可设置为 RTT 的 0.5-1.5 倍</strong>。如果心跳间隔太短，etcd 实例会频繁发送没必要的心跳，增加 CPU 和网络的使用率。另外，过长的心跳间隔也会延长选举超时时间，一旦选举超时过长，还会导致需要更长的时间才能发现 Leader 故障。测量 RTT 最简单方法就是用 <strong>PING 工具</strong>。</p>
<p>对于选举超时的时间，应该<strong>基于心跳间隔和节点的平均 RTT 去设置</strong>。选举超时应该至少是 RTT 的 10 倍，这样才能视为在该网络中容错。例如，节点间的 RTT 是 10ms，那么超时时间至少应该是 100ms。</p>
<p>选举超时时间最大限制是 50000ms（即 50s），只有 etcd 被部署在全球范围内时，才应该使用这个值。如果出现不均匀的网络性能或者常规的网络延迟和丢失，会引起多次 etcd 网络重试，所以 5s 是一个安全的 RTT 最高值。只有心跳间隔为 5s 时，超时时间才应该设置为 50s。</p>
<h1 id="4-小结"><a href="#4-小结" class="headerlink" title="4 小结"></a>4 小结</h1><p>除了服务端的优化，我们在日常使用过程中还要注意客户端的使用，正确的用法对于一个组件来说很重要。从实践角度来说，etcd 多用于<strong>读多写少</strong>的场景，读写的开销不一样，我们应该尽量避免频繁更新键值对数据。除此之外，我们还应该尽可能地复用 lease，避免重复创建 lease。对于相同 TTL 失效时间的键值对，绑定到相同的 lease 租约上也可以避免大量重复创建 lease。</p>

      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color5">etcd</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/etcd//" class="article-tag-list-link color5">etcd</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/etcd/etcd%E6%97%A5%E5%B8%B8%E8%BF%90%E7%BB%B4%EF%BC%9A%E9%9B%86%E7%BE%A4%E8%B0%83%E4%BC%98/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-etcd/etcd概念：etcd的架构" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/etcd/etcd%E6%A6%82%E5%BF%B5%EF%BC%9Aetcd%E7%9A%84%E6%9E%B6%E6%9E%84/">etcd概念:etcd的架构</a>
    </h1>
  

        
        <a href="/2021/04/28/etcd/etcd%E6%A6%82%E5%BF%B5%EF%BC%9Aetcd%E7%9A%84%E6%9E%B6%E6%9E%84/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h1 id="1-etcd-项目结构"><a href="#1-etcd-项目结构" class="headerlink" title="1 etcd 项目结构"></a>1 etcd 项目结构</h1><p>在介绍 etcd 整体的架构之前，我们先来看一下 etcd 项目代码的目录结构：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">$</span><span class="bash"> tree</span></span><br><span class="line"></span><br><span class="line">.</span><br><span class="line"></span><br><span class="line">├── auth</span><br><span class="line"></span><br><span class="line">├── build</span><br><span class="line"></span><br><span class="line">├── client</span><br><span class="line"></span><br><span class="line">├── clientv3</span><br><span class="line"></span><br><span class="line">├── contrib</span><br><span class="line"></span><br><span class="line">├── embed</span><br><span class="line"></span><br><span class="line">├── etcdctl</span><br><span class="line"></span><br><span class="line">├── etcdmain</span><br><span class="line"></span><br><span class="line">├── etcdserver</span><br><span class="line"></span><br><span class="line">├── functional</span><br><span class="line"></span><br><span class="line">├── hack</span><br><span class="line"></span><br><span class="line">├── integration</span><br><span class="line"></span><br><span class="line">├── lease</span><br><span class="line"></span><br><span class="line">├── logos</span><br><span class="line"></span><br><span class="line">├── mvcc</span><br><span class="line"></span><br><span class="line">├── pkg</span><br><span class="line"></span><br><span class="line">├── proxy</span><br><span class="line"></span><br><span class="line">├── raft</span><br><span class="line"></span><br><span class="line">├── scripts</span><br><span class="line"></span><br><span class="line">├── security</span><br><span class="line"></span><br><span class="line">├── tests</span><br><span class="line"></span><br><span class="line">├── tools</span><br><span class="line"></span><br><span class="line">├── vendor</span><br><span class="line"></span><br><span class="line">├── version</span><br><span class="line"></span><br><span class="line">└── wal</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<p>可以看到，etcd 的包还是挺多的，有二十多个。接下来我们来具体分析下其中每一个包的职责定义，整理之后如下表所示：</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/%E7%AC%94%E8%AE%B0/image-20210423162257218.png" alt="image-20210423162257218"></p>
<p>etcd 核心的模块有 lease、mvcc、raft、etcdserver，其余都是辅助的功能。其中 <strong>etcdserver 是其他模块的整合</strong>。</p>
<h1 id="2-etcd-整体架构"><a href="#2-etcd-整体架构" class="headerlink" title="2 etcd 整体架构"></a>2 etcd 整体架构</h1><p>接下来，我们看看 etcd 的整体架构。我在上面的 etcd 项目总览中提到了 etcd 中核心的几个模块，我们使用分层的方式来描绘 etcd 的架构图，如下所示：</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/%E7%AC%94%E8%AE%B0/image-20210423162231953.png" alt="image-20210423162231953"></p>
<ul>
<li><strong>客户端层</strong>：包括 clientv3 和 etcdctl 等客户端。用户通过命令行或者客户端调用提供了 RESTful 风格的 API，降低了 etcd 的使用复杂度。除此之外，客户端层的负载均衡（etcd V3.4 版本的客户端默认使用的是 Round-robin，即轮询调度）和节点间故障转移等特性，提升了 etcd 服务端的高可用性。<strong>需要注意的是，etcd V3.4 之前版本的客户端存在负载均衡的 Bug，如果第一个节点出现异常，访问服务端时也可能会出现异常，建议进行升级。</strong></li>
<li><strong>API 接口层</strong>：API 接口层提供了客户端访问服务端的通信协议和接口定义，以及服务端节点之间相互通信的协议，我将会在下一讲重点讲解 etcd 的通信接口。etcd 有 V3 和 V2 两个版本。etcd V3 使用 gRPC 作为消息传输协议；对于之前的 V2 版本，etcd 默认使用 HTTP/1.x 协议。对于不支持 gRPC 的客户端语言，etcd 提供 JSON 的 grpc-gateway。通过 grpc-gateway 提供 RESTful 代理，转换 HTTP/JSON 请求为 gRPC 的 Protocol Buffer 格式的消息。</li>
<li><strong>etcd Raft 层</strong>：负责 Leader 选举和日志复制等功能，除了与本节点的 etcd Server 通信之外，还与集群中的其他 etcd 节点进行交互，实现<strong>分布式一致性数据同步</strong>的关键工作。</li>
<li><strong>逻辑层</strong>：etcd 的业务逻辑层，包括鉴权、租约、KVServer、MVCC 和 Compactor 压缩等核心功能特性。</li>
<li><strong>etcd 存储</strong>：实现了快照、预写式日志 WAL（Write Ahead Log）。etcd V3 版本中，使用 BoltDB 来持久化存储集群元数据和用户写入的数据。</li>
</ul>
<h1 id="3-etcd-交互总览"><a href="#3-etcd-交互总览" class="headerlink" title="3 etcd 交互总览"></a>3 etcd 交互总览</h1><p>我们通过学习 etcd 服务端处理客户端的写请求的过程，展示 etcd 内部各个模块之间的交互。首先通过命令行工具 etcdctl 写入键值对：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">etcdctl --endpoints http:&#x2F;&#x2F;127.0.0.1:2379 put foo bar</span><br></pre></td></tr></table></figure>

<p>下图中展示了 etcd 处理一个客户端请求涉及的模块和流程。</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/%E7%AC%94%E8%AE%B0/image-20210423162431790.png" alt="image-20210423162431790"></p>
<p>从上至下依次为客户端 → API 接口层 → etcd Server → etcd raft 算法库。我们根据请求处理的过程，将 etcd Server 和 etcd raft 算法库单独说明。</p>
<ul>
<li><p><strong>etcd Server</strong>：接收客户端的请求，在上述的 etcd 项目代码中对应 etcdserver 包。请求到达 etcd Server 之后，经过 KVServer 拦截，实现诸如日志、Metrics 监控、请求校验等功能。etcd Server 中的 raft 模块，用于与 etcd-raft 库进行通信。applierV3 模块封装了 etcd V3 版本的数据存储；WAL 用于写数据日志，WAL 中保存了任期号、投票信息、已提交索引、提案内容等，etcd 根据 WAL 中的内容在启动时恢复，以此实现集群的数据一致性。</p>
</li>
<li><p><strong>etcdraft</strong>：etcd 的 raft 库。raftLog 用于管理 raft 协议中单个节点的日志，都处于内存中。raftLog 中还有两种结构体 unstable 和 storage，unsable 中存储不稳定的数据，表示还没有 commit，而 storage 中都是已经被 commit 了的数据。这两种结构体分别用于不同步骤的存储，我们将在下面的交互流程中介绍。除此之外，raft 库更重要的是负责与集群中的其他 etcd Server 进行交互，实现分布式一致性。</p>
</li>
</ul>
<p>在上图中，客户端请求与 etcd 集群交互包括如下两个步骤：</p>
<ul>
<li><p>首先是写数据到 etcd 节点中；</p>
</li>
<li><p>其次是当前的 etcd 节点与集群中的其他 etcd 节点之间进行通信，确认存储数据成功之后回复客户端。</p>
</li>
</ul>
<p>请求流程可划分为以下的子步骤：</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/%E7%AC%94%E8%AE%B0/image-20210423162505011.png" alt="image-20210423162505011"></p>
<ul>
<li><p>客户端通过负载均衡算法选择一个 etcd 节点，发起 gRPC 调用；</p>
</li>
<li><p>etcd Server 收到客户端请求；</p>
</li>
<li><p>经过 gRPC 拦截、Quota 校验，Quota 模块用于校验 etcd db 文件大小是否超过了配额；</p>
</li>
<li><p>接着 KVServer 模块将请求发送给本模块中的 raft，这里负责与 etcd raft 模块进行通信；</p>
</li>
<li><p>发起一个提案，命令为<code>put foo bar</code>，即使用 put 方法将 foo 更新为 bar；</p>
</li>
<li><p>在 raft 中会将数据封装成 raft 日志的形式提交给 raft 模块；</p>
</li>
<li><p>raft 模块会首先保存到 raftLog 的 unstable 存储部分；</p>
</li>
<li><p>raft 模块通过 raft 协议与集群中其他 etcd 节点进行交互。</p>
</li>
</ul>
<p>需要注意的是，在 raft 协议中写入数据的 etcd 必定是 leader 节点，如果客户端提交数据到非 leader 节点时，该节点需要将请求转发到 etcd leader 节点处理。</p>
<p>我们继续来看相应的应答步骤，流程如下：</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/%E7%AC%94%E8%AE%B0/image-20210423162524175.png" alt="image-20210423162524175"></p>
<ul>
<li><p>提案通过 RaftHTTP 网络模块转发，集群中的其他节点接收到该提案；</p>
</li>
<li><p>在收到提案之后，集群中其他节点向 leader 节点应答 “我已经接收这条日志数据”；</p>
</li>
<li><p>Leader 收到应答之后，统计应答的数量，当满足超过集群半数以上节点，应答接收成功；</p>
</li>
<li><p>etcd raft 算法模块构造 Ready 结构体，用来通知 etcd Server 模块，该日志数据已经被 commit；</p>
</li>
<li><p>etcd Server 中的 raft 模块（交互图中有标识），收到 Ready 消息后，会将这条日志数据写入到 WAL 模块中；</p>
</li>
<li><p>正式通知 etcd Server 该提案已经被 commit；</p>
</li>
<li><p>etcd Server 调用 applierV3 模块，将日志写入持久化存储中；</p>
</li>
<li><p>etcd Server 应答客户端该数据写入成功；</p>
</li>
<li><p>etcd Server 调用 etcd raft 库，将这条日志写入到 raftLog 模块中的 storage。</p>
</li>
</ul>
<p>上述过程中，提案经过网络转发，<strong>当多数 etcd 节点持久化日志数据成功并进行应答，提案的状态会变成已提交</strong>。</p>
<p>在应答某条日志数据是否已经 commit 时，为什么 etcd raft 模块首先写入到 WAL 模块中？这是因为该过程仅仅添加一条日志，一方面开销小，速度会很快；另一方面，如果在后面 applierV3 写入失败，etcd 服务端在重启的时候也可以根据 WAL 模块中的日志数据进行恢复。etcd Server 从 raft 模块获取已提交的日志条目，由 applierV3 模块通过 MVCC 模块执行提案内容，更新状态机。</p>
<p>整个过程中，etcd raft 模块中的 raftLog 数据在内存中存储，在服务重启后失效；客户端请求的数据则被持久化保存到 WAL 和 applierV3 中，不会在重启之后丢失。</p>

      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color5">etcd</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/etcd//" class="article-tag-list-link color5">etcd</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/etcd/etcd%E6%A6%82%E5%BF%B5%EF%BC%9Aetcd%E7%9A%84%E6%9E%B6%E6%9E%84/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-etcd/etcd概念：核心基础入门" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/etcd/etcd%E6%A6%82%E5%BF%B5%EF%BC%9A%E6%A0%B8%E5%BF%83%E5%9F%BA%E7%A1%80%E5%85%A5%E9%97%A8/">etcd概念:核心基础入门</a>
    </h1>
  

        
        <a href="/2021/04/28/etcd/etcd%E6%A6%82%E5%BF%B5%EF%BC%9A%E6%A0%B8%E5%BF%83%E5%9F%BA%E7%A1%80%E5%85%A5%E9%97%A8/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h1 id="1-为什么需要-etcd"><a href="#1-为什么需要-etcd" class="headerlink" title="1 为什么需要 etcd"></a>1 为什么需要 etcd</h1><p>在分布式环境中，由于网络的复杂性、不确定性以及节点故障等情况，会产生一系列的问题。最常见的、最大的难点就是<strong>数据存储不一致</strong>的问题，即多个服务实例自身的数据或者获取到的数据各不相同。因此我们需要基于一致性的存储组件构建可靠的分布式系统。</p>
<h2 id="1-1-分布式中的-CAP-理论"><a href="#1-1-分布式中的-CAP-理论" class="headerlink" title="1.1 分布式中的 CAP 理论"></a>1.1 分布式中的 CAP 理论</h2><p>CAP 原理是描述分布式系统下节点数据同步的基本定理，分别指 <strong>Consistency（一致性）、Availability（可用性）和 Partition tolerance（分区容错性）</strong>，这三个要素最多只能同时实现两点，不能三者兼顾。</p>
<p>基于分布式系统的基本特质，P（分区容错性）是必须要满足的，所以接下来需要考虑满足 C（一致性）还是 A（可用性）。</p>
<h2 id="1-2-etcd-是什么"><a href="#1-2-etcd-是什么" class="headerlink" title="1.2 etcd 是什么"></a>1.2 etcd 是什么</h2><p>根据 <a target="_blank" rel="noopener" href="https://etcd.io/">etcd 官网</a>的介绍，有如下定义：</p>
<blockquote>
<p>A highly-available key value store for shared configuration and service discovery.<br>即一个用于配置共享和服务发现的键值存储系统。</p>
</blockquote>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/%E7%AC%94%E8%AE%B0/image-20210423155624052.png" alt="image-20210423155624052"></p>
<ul>
<li><p>etcd 归根结底是一个存储组件，且可以实现配置共享和服务发现。</p>
</li>
<li><p>etcd 就是这样一款<strong>实现了元数据信息可靠存储的组件</strong>。</p>
</li>
<li><p><strong>etcd 可集中管理配置信息</strong>。服务端将配置信息存储于 etcd，客户端通过 etcd 得到服务配置信息，etcd 监听配置信息的改变，发现改变通知客户端。</p>
</li>
<li><p>etcd <strong>满足 CAP 理论中的 CP</strong>（一致性和分区容错性） 指标，由此我们知道，etcd 解决了分布式系统中一致性存储的问题。</p>
</li>
</ul>
<h2 id="1-3-etcd-中常用的术语"><a href="#1-3-etcd-中常用的术语" class="headerlink" title="1.3 etcd 中常用的术语"></a>1.3 etcd 中常用的术语</h2><p>为了我们接下来更好地学习 etcd，我在这里给你列举了常用的 etcd 术语，尽快熟悉它们也会对接下来的学习有所助益。</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/%E7%AC%94%E8%AE%B0/image-20210423155839668.png" alt="image-20210423155839668"></p>
<h2 id="1-4-etcd-的特性"><a href="#1-4-etcd-的特性" class="headerlink" title="1.4 etcd 的特性"></a>1.4 etcd 的特性</h2><p>etcd 可以用来构建高可用的分布式键值数据库，总结来说有如下特点。</p>
<ul>
<li><p><strong>简单</strong>：etcd 的安装简单，且为用户提供了 HTTP API，使用起来也很简单。</p>
</li>
<li><p><strong>存储</strong>：etcd 的基本功能，数据分层存储在文件目录中，类似于我们日常使用的文件系统。</p>
</li>
<li><p><strong>Watch 机制</strong>：Watch 指定的键、前缀目录的更改，并对更改时间进行通知。</p>
</li>
<li><p><strong>安全通信</strong>：支持 SSL 证书验证。</p>
</li>
<li><p><strong>高性能</strong>：etcd 单实例可以支持 2K/s 读操作，官方也有提供基准测试脚本。</p>
</li>
<li><p><strong>一致可靠</strong>：基于 Raft 共识算法，实现分布式系统内部数据存储、服务调用的一致性和高可用性。</p>
</li>
</ul>
<p>etcd 是一个实现了分布式一致性键值对存储的中间件。etcd 集群中的节点基于 Raft 算法进行通信，Raft 算法保证了微服务实例或机器集群所访问的数据的可靠一致性。</p>
<p>在分布式系统或者 Kubernetes 集群中，etcd 可以<strong>作为服务注册与发现和键值对存储组件</strong>。</p>
<h2 id="1-5-etcd-的应用场景"><a href="#1-5-etcd-的应用场景" class="headerlink" title="1.5 etcd 的应用场景"></a>1.5 etcd 的应用场景</h2><p>etcd 在稳定性、可靠性和可伸缩性上表现极佳，同时也为云原生应用系统提供了协调机制。etcd 经常用于服务注册与发现的场景，此外还有键值对存储、消息发布与订阅、分布式锁等场景。</p>
<h3 id="1-5-1-键值对存储"><a href="#1-5-1-键值对存储" class="headerlink" title="1.5.1 键值对存储"></a>1.5.1 <strong>键值对存储</strong></h3><p>etcd 是一个用于<strong>键值存储</strong>的组件，存储是 etcd 最基本的功能，其他应用场景都建立在 etcd 的可靠存储上。比如 Kubernetes 将一些元数据存储在 etcd 中，将存储状态数据的复杂工作交给 etcd，Kubernetes 自身的功能和架构就能更加稳定。</p>
<p>etcd 基于 Raft 算法，能够有力地保证分布式场景中的一致性。各个服务启动时注册到 etcd 上，同时为这些服务配置键的 TTL 时间。注册到 etcd 上面的各个服务实例通过心跳的方式定期续租，实现服务实例的状态监控。</p>
<h3 id="1-5-2-消息发布与订阅"><a href="#1-5-2-消息发布与订阅" class="headerlink" title="1.5.2 消息发布与订阅"></a>1.5.2 <strong>消息发布与订阅</strong></h3><p>在分布式系统中，服务之间还可以通过消息通信，即消息的发布与订阅，如下图所示：</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/%E7%AC%94%E8%AE%B0/image-20210423160254638.png" alt="image-20210423160254638"></p>
<p>通过构建 etcd 消息中间件，服务提供者发布对应主题的消息，消费者则订阅他们关心的主题，一旦对应的主题有消息发布，就会产生订阅事件，消息中间件就会通知该主题所有的订阅者。</p>
<h3 id="1-5-3-分布式锁"><a href="#1-5-3-分布式锁" class="headerlink" title="1.5.3 分布式锁"></a>1.5.3 <strong>分布式锁</strong></h3><p>分布式系统中涉及多个服务实例，存在跨进程之间资源调用，对于资源的协调分配，单体架构中的锁已经无法满足需要，需要引入分布式锁的概念。etcd 基于 Raft 算法，实现分布式集群的一致性，存储到 etcd 集群中的值必然是全局一致的，因此基于 etcd 很容易实现分布式锁。</p>
<h2 id="1-6-etcd-的核心架构"><a href="#1-6-etcd-的核心架构" class="headerlink" title="1.6 etcd 的核心架构"></a>1.6 etcd 的核心架构</h2><p>tcd 有 etcd Server、gRPC Server、存储相关的 MVCC 、Snapshot、WAL，以及 Raft 模块。</p>
<p>其中：</p>
<ul>
<li><p>etcd Server 用于对外接收和处理客户端的请求；</p>
</li>
<li><p>gRPC Server 则是 etcd 与其他 etcd 节点之间的通信和信息同步；</p>
</li>
<li><p>MVCC，即多版本控制，etcd 的存储模块，键值对的每一次操作行为都会被记录存储，这些数据底层存储在 BoltDB 数据库中；</p>
</li>
<li><p>WAL，预写式日志，etcd 中的数据提交前都会记录到日志；</p>
</li>
<li><p>Snapshot 快照，以防 WAL 日志过多，用于存储某一时刻 etcd 的所有数据；</p>
</li>
<li><p>Snapshot 和 WAL 相结合，etcd 可以有效地进行数据存储和节点故障恢复等操作。</p>
</li>
</ul>
<p>etcd 内部实现机制复杂，但etcd 中的数据结构很简单,它的数据存储其实就是键值对的有序映射。我们可以通过 <strong>etcdctl 客户端命令行</strong>操作和访问 etcd 中的数据，或者通过 <strong>HTTP API</strong> 接口直接访问 etcd。</p>
<p>etcd 还提供了一种键值对监测机制，即 Watch 机制，客户端通过订阅相关的键值对，获取其更改的事件信息。Watch 机制实时获取 etcd 中的增量数据更新，使数据与 etcd 同步。</p>
<h2 id="1-7-版本差异"><a href="#1-7-版本差异" class="headerlink" title="1.7 版本差异"></a>1.7 版本差异</h2><p>etcd 目前有 V2.x 和 V3.x 两个大版本。etcd V2 和 V3 是在底层使用同一套 Raft 算法的两个独立应用，但相互之间实现原理和使用方法上差别很大，接口不一样、存储不一样，两个版本的数据互相隔离。</p>
<p>至于由 etcd V2 升级到 etcd V3 的情况，原有数据只能通过 etcd V2 接口访问，V3 接口创建的数据只能通过新的 V3 的接口访问。我们的专栏重点讲解<strong>当前常用且主流的 V3 版本</strong>。</p>
<h1 id="2-小结"><a href="#2-小结" class="headerlink" title="2 小结"></a>2 小结</h1><p>关于 etcd 需要记住以下三点：</p>
<ul>
<li><p>etcd 是云原生架构中的存储基石，可以有效保证存储数据的一致性和可靠性；</p>
</li>
<li><p>etcd 内部实现机制复杂，但是对外提供了简单直接的 API 接口；</p>
</li>
<li><p>使用 etcd 的常见分布式场景包括键值对存储、服务注册与发现、消息订阅与发布、分布式锁等。</p>
</li>
</ul>

      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color5">etcd</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/etcd//" class="article-tag-list-link color5">etcd</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/etcd/etcd%E6%A6%82%E5%BF%B5%EF%BC%9A%E6%A0%B8%E5%BF%83%E5%9F%BA%E7%A1%80%E5%85%A5%E9%97%A8/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-kubernetes/K8s资源调度Statefulset" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/kubernetes/K8s%E8%B5%84%E6%BA%90%E8%B0%83%E5%BA%A6Statefulset/">k8s资源调度Statefulset</a>
    </h1>
  

        
        <a href="/2021/04/28/kubernetes/K8s%E8%B5%84%E6%BA%90%E8%B0%83%E5%BA%A6Statefulset/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h2 id="1-1-有状态应用管理StatefulSet"><a href="#1-1-有状态应用管理StatefulSet" class="headerlink" title="1.1  有状态应用管理StatefulSet"></a>1.1  有状态应用管理StatefulSet</h2><p>StatefulSet（有状态集，缩写为sts）常用于部署有状态的且需要有序启动的应用程序</p>
<p>StatefulSet 这个控制器的主要作用之一，就是使用 Pod 模板创建 Pod 的时候，对它们进行编号，并且按照编号顺序逐一完成创建工作。而当 StatefulSet 的“控制循环”发现 Pod 的“实际状态”与“期望状态”不一致，需要新建或者删除 Pod 进行“调谐”的时候，它会严格按照这些 Pod 编号的顺序，逐一完成这些操作。</p>
<p><strong>StatefulSet 的核心功能，就是通过某种方式记录这些状态，然后在 Pod 被重新创建时，能够为新 Pod 恢复这些状态。</strong></p>
<h3 id="1-1-1-StatefulSet的基本概念"><a href="#1-1-1-StatefulSet的基本概念" class="headerlink" title="1.1.1  StatefulSet的基本概念"></a>1.1.1  StatefulSet的基本概念</h3><p>StatefulSet主要用于管理有状态应用程序的工作负载API对象。比如在生产环境中，可以部署ElasticSearch集群、MongoDB集群或者需要持久化的RabbitMQ集群、Redis集群、Kafka集群和ZooKeeper集群等。</p>
<p>和Deployment类似，一个StatefulSet也同样管理着基于相同容器规范的Pod。不同的是，StatefulSet为每个Pod维护了一个粘性标识。这些Pod是根据相同的规范创建的，但是不可互换，每个Pod都有一个持久的标识符，在重新调度时也会保留，一般格式为StatefulSetName-Number。比如定义一个名字是Redis-Sentinel的StatefulSet，指定创建三个Pod，那么创建出来的Pod名字就为Redis-Sentinel-0、Redis-Sentinel-1、Redis-Sentinel-2。</p>
<p>而StatefulSet创建的Pod一般使用Headless Service（无头服务）进行通信，和普通的Service的区别在于Headless Service没有ClusterIP，它使用的是Endpoint进行互相通信.</p>
<p>Headless一般的格式为：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">statefulSetName-&#123;0..N-1&#125;.serviceName.namespace.svc.cluster.local</span><br></pre></td></tr></table></figure>

<p>说明：</p>
<ul>
<li><p>serviceName为Headless Service的名字。</p>
</li>
<li><p>0..N-1为Pod所在的序号，从0开始到N-1。</p>
</li>
<li><p>statefulSetName为StatefulSet的名字。</p>
</li>
<li><p>namespace为服务所在的命名空间。</p>
</li>
<li><p>.cluster.local为Cluster Domain（集群域）。</p>
</li>
</ul>
<h3 id="1-1-2-应用场景举例"><a href="#1-1-2-应用场景举例" class="headerlink" title="1.1.2 应用场景举例"></a>1.1.2 应用场景举例</h3><p>假如公司某个项目需要在Kubernetes中部署一个主从模式的Redis，此时使用StatefulSet部署就极为合适，因为StatefulSet启动时，只有当前一个容器完全启动时，后一个容器才会被调度，并且每个容器的标识符是固定的，那么就可以通过标识符来断定当前Pod的角色。</p>
<p>比如用一个名为redis-ms的StatefulSet部署主从架构的Redis，第一个容器启动时，它的标识符为redis-ms-0，并且Pod内主机名也为redis-ms-0，此时就可以根据主机名来判断，当主机名为redis-ms-0的容器作为Redis的主节点，其余从节点，那么Slave连接Master主机配置就可以使用不会更改的Master的Headless Service，此时Redis从节点（Slave）配置文件如下：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">port 6379</span><br><span class="line">slaveof  redis-sentinel-master-ss-0.redis-sentinel-master-ss.public-service.svc.cluster.local 6379</span><br><span class="line">tcp-backlog 511</span><br><span class="line">timeout 0</span><br><span class="line">tcp-keepalive 0</span><br></pre></td></tr></table></figure>

<p>其中redis-ms-0.redis-ms.public-service.svc.cluster.local是Redis Master的Headless Service，在同一命名空间下只需要写redis-ms-0.redis-ms即可，后面的public-service.svc.cluster.local可以省略。</p>
<h3 id="1-1-3-StatefulSet注意事项"><a href="#1-1-3-StatefulSet注意事项" class="headerlink" title="1.1.3 StatefulSet注意事项"></a>1.1.3 StatefulSet注意事项</h3><p>一般StatefulSet用于有以下一个或者多个需求的应用程序：</p>
<ul>
<li><p>需要稳定的独一无二的网络标识符。</p>
</li>
<li><p>需要持久化数据。</p>
</li>
<li><p>需要有序的、优雅的部署和扩展。</p>
</li>
<li><p>需要有序的自动滚动更新。</p>
</li>
</ul>
<p>如果应用程序不需要任何稳定的标识符或者有序的部署、删除或者扩展，应该使用无状态的控制器部署应用程序，比如Deployment或者ReplicaSet。</p>
<p>StatefulSet是Kubernetes 1.9版本之前的beta资源，在1.5版本之前的任何Kubernetes版本都没有。</p>
<p>Pod所用的存储必须由PersistentVolume Provisioner（持久化卷配置器）根据请求配置StorageClass，或者由管理员预先配置，当然也可以不配置存储。</p>
<p>为了确保数据安全，删除和缩放StatefulSet不会删除与StatefulSet关联的卷，可以手动选择性地删除PVC和PV（参考后续关于PV和PVC）。</p>
<p>StatefulSet目前使用Headless Service（无头服务）负责Pod的网络身份和通信，需要提前创建此服务。</p>
<p>删除一个StatefulSet时，不保证对Pod的终止，要在StatefulSet中实现Pod的有序和正常终止，可以在删除之前将StatefulSet的副本缩减为0。</p>
<h3 id="1-1-4-创建一个StatefulSet"><a href="#1-1-4-创建一个StatefulSet" class="headerlink" title="1.1.4 创建一个StatefulSet"></a>1.1.4 创建一个StatefulSet</h3><h4 id="1-1-4-1-定义一个简单的StatefulSet的示例如下："><a href="#1-1-4-1-定义一个简单的StatefulSet的示例如下：" class="headerlink" title="1.1.4.1 定义一个简单的StatefulSet的示例如下："></a>1.1.4.1 定义一个简单的StatefulSet的示例如下：</h4><figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br></pre></td><td class="code"><pre><span class="line"><span class="string">cat</span> <span class="string">&lt;&lt;EOF&gt;</span> <span class="string">sts-web.yaml</span> </span><br><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Service</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">nginx</span></span><br><span class="line">  <span class="attr">labels:</span></span><br><span class="line">    <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">ports:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">port:</span> <span class="number">80</span></span><br><span class="line">    <span class="attr">name:</span> <span class="string">web</span></span><br><span class="line">  <span class="attr">clusterIP:</span> <span class="string">None</span></span><br><span class="line">  <span class="attr">selector:</span></span><br><span class="line">    <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line"><span class="meta">---</span></span><br><span class="line"><span class="attr">apiVersion:</span> <span class="string">apps/v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">StatefulSet</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">web</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">serviceName:</span> <span class="string">&quot;nginx&quot;</span></span><br><span class="line">  <span class="attr">replicas:</span> <span class="number">3</span></span><br><span class="line">  <span class="attr">selector:</span></span><br><span class="line">    <span class="attr">matchLabels:</span></span><br><span class="line">      <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line">  <span class="attr">template:</span></span><br><span class="line">    <span class="attr">metadata:</span></span><br><span class="line">      <span class="attr">labels:</span></span><br><span class="line">        <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line">    <span class="attr">spec:</span></span><br><span class="line">      <span class="attr">containers:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">nginx</span></span><br><span class="line">        <span class="attr">image:</span> <span class="string">nginx:1.15.2</span></span><br><span class="line">        <span class="attr">ports:</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">containerPort:</span> <span class="number">80</span></span><br><span class="line">          <span class="attr">name:</span> <span class="string">web</span></span><br><span class="line"><span class="string">EOF</span></span><br></pre></td></tr></table></figure>

<p><strong>注意：此示例没有添加存储配置，后面的会单独讲解存储相关的知识点</strong></p>
<p>其中：</p>
<ul>
<li><p>kind: Service定义了一个名字为Nginx的Headless Service，创建的Service格式为nginx-0.nginx.default.svc.cluster.local，其他的类似，因为没有指定Namespace（命名空间），所以默认部署在default。</p>
</li>
<li><p>kind: StatefulSet定义了一个名字为web的StatefulSet，replicas表示部署Pod的副本数，本实例为2。</p>
</li>
</ul>
<p><strong>在StatefulSet中必须设置Pod选择器（.spec.selector）用来匹配其标签（.spec.template.metadata.labels）。在1.8版本之前，如果未配置该字段（.spec.selector），将被设置为默认值，在1.8版本之后，如果未指定匹配Pod Selector，则会导致StatefulSet创建错误。</strong></p>
<p><strong>当StatefulSet控制器创建Pod时，它会添加一个标签statefulset.kubernetes.io/pod-name，该标签的值为Pod的名称，用于匹配Service。</strong></p>
<h4 id="1-1-4-2-创建Statefulset"><a href="#1-1-4-2-创建Statefulset" class="headerlink" title="1.1.4.2 创建Statefulset"></a>1.1.4.2 创建Statefulset</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f sts-web.yaml </span><br></pre></td></tr></table></figure>



<h4 id="1-1-4-3-查看Statefulset"><a href="#1-1-4-3-查看Statefulset" class="headerlink" title="1.1.4.3 查看Statefulset"></a>1.1.4.3 查看Statefulset</h4><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get sts</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20201210220651360.png" alt="image-20201210220651360"></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get po</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20201210221311802.png" alt="image-20201210221311802"></p>
<p>可以看到pod的命名规则是根据Statfulset的名称加上数字从0开始递增的有规律命名。</p>
<h4 id="1-1-4-4-访问Statefulset"><a href="#1-1-4-4-访问Statefulset" class="headerlink" title="1.1.4.4 访问Statefulset"></a>1.1.4.4 访问Statefulset</h4><ul>
<li><p>创建一个busybox的pod</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line"><span class="string">cat&lt;&lt;EOF</span> <span class="string">|</span> <span class="string">kubectl</span> <span class="string">apply</span> <span class="string">-f</span> <span class="bullet">-</span></span><br><span class="line"><span class="attr">apiVersion:</span> <span class="string">v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">Pod</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">busybox</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">default</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">containers:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">busybox</span></span><br><span class="line">    <span class="attr">image:</span> <span class="string">busybox:1.28</span></span><br><span class="line">    <span class="attr">command:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="string">sleep</span></span><br><span class="line">      <span class="bullet">-</span> <span class="string">&quot;3600&quot;</span></span><br><span class="line">    <span class="attr">imagePullPolicy:</span> <span class="string">IfNotPresent</span></span><br><span class="line">  <span class="attr">restartPolicy:</span> <span class="string">Always</span></span><br><span class="line"><span class="string">EOF</span></span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>进入busybox中</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl exec -ti busybox -- sh</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>访问创建的名为web的Statefulset</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">nslookup web-0.nginx</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20201210220955582.png" alt="image-20201210220955582"></p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">ping web-2.nginx</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20201210221538084.png" alt="image-20201210221538084"></p>
</li>
</ul>
<p>经过测试可以发现，在容器内部可以使用<code>web-0.nginx</code>这样的方式直接访问到Statefulset部署的pod的IP</p>
<h3 id="1-1-5-StatefulSet扩缩容"><a href="#1-1-5-StatefulSet扩缩容" class="headerlink" title="1.1.5 StatefulSet扩缩容"></a>1.1.5 StatefulSet扩缩容</h3><p>和Deployment类似，可以通过更新replicas字段扩容/缩容StatefulSet，也可以使用kubectl scale或者kubectl patch来扩容/缩容一个StatefulSet。</p>
<h4 id="1-1-5-1-扩缩容过程概述"><a href="#1-1-5-1-扩缩容过程概述" class="headerlink" title="1.1.5.1 扩缩容过程概述"></a>1.1.5.1 扩缩容过程概述</h4><p>Statefulset在扩缩容过程中，严格按照Statefulset的id数字大小顺序进行调度，如</p>
<ul>
<li><p>扩容时</p>
<p><strong>会从序号最小的pod开始0…1…2…3，依次创建，中途出现序号较小的pod被删除等情况，会在当前pod创建完后，回去先将较小的序号创建完毕后再继续往下创建。</strong></p>
</li>
<li><p>缩容时</p>
<p><strong>会从序号最大的pod开始一次删除，中途出现序号较小的pod被提前删除的情况时，会暂停当前的删除动作，等较小序号的pod恢复后继续</strong></p>
</li>
</ul>
<h4 id="1-1-5-2-扩缩容示例"><a href="#1-1-5-2-扩缩容示例" class="headerlink" title="1.1.5.2 扩缩容示例"></a>1.1.5.2 扩缩容示例</h4><ul>
<li><p>使用kubectl scale将Statefulset扩容到5个</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl scale sts web --replicas&#x3D;5</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>使用kubectl patch将副本数改为3</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl patch sts web -p &#39;&#123;&quot;spec&quot;:&#123;&quot;replicas&quot;:3&#125;&#125;&#39;</span><br></pre></td></tr></table></figure>



</li>
</ul>
<h3 id="1-1-6-更新策略"><a href="#1-1-6-更新策略" class="headerlink" title="1.1.6 更新策略"></a>1.1.6 更新策略</h3><h4 id="1-1-6-1-On-Delete策略"><a href="#1-1-6-1-On-Delete策略" class="headerlink" title="1.1.6.1 On Delete策略"></a>1.1.6.1 On Delete策略</h4><p>OnDelete更新策略实现了传统（1.7版本之前）的行为，它也是默认的更新策略。当我们选择这个更新策略并修改StatefulSet的.spec.template字段时，StatefulSet控制器不会自动更新Pod，<strong>我们必须手动删除Pod才能使控制器创建新的Pod</strong>。</p>
<ul>
<li>yaml文件中定义方式如下</li>
</ul>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">apps/v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">StatefulSet</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">generation:</span> <span class="number">5</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">web</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">default</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">podManagementPolicy:</span> <span class="string">OrderedReady</span></span><br><span class="line">  <span class="attr">replicas:</span> <span class="number">3</span></span><br><span class="line">  <span class="attr">revisionHistoryLimit:</span> <span class="number">10</span></span><br><span class="line">  <span class="attr">selector:</span></span><br><span class="line">    <span class="attr">matchLabels:</span></span><br><span class="line">      <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line">  <span class="attr">serviceName:</span> <span class="string">nginx</span></span><br><span class="line">  <span class="attr">template:</span></span><br><span class="line">    <span class="attr">metadata:</span></span><br><span class="line">      <span class="attr">creationTimestamp:</span> <span class="literal">null</span></span><br><span class="line">      <span class="attr">labels:</span></span><br><span class="line">        <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line">    <span class="attr">spec:</span></span><br><span class="line">      <span class="attr">containers:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">image:</span> <span class="string">nginx:1.15.2</span></span><br><span class="line">        <span class="attr">imagePullPolicy:</span> <span class="string">IfNotPresent</span></span><br><span class="line">        <span class="attr">name:</span> <span class="string">nginx</span></span><br><span class="line">        <span class="attr">ports:</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">containerPort:</span> <span class="number">80</span></span><br><span class="line">          <span class="attr">name:</span> <span class="string">web</span></span><br><span class="line">          <span class="attr">protocol:</span> <span class="string">TCP</span></span><br><span class="line">        <span class="attr">resources:</span> &#123;&#125;</span><br><span class="line">        <span class="attr">terminationMessagePath:</span> <span class="string">/dev/termination-log</span></span><br><span class="line">        <span class="attr">terminationMessagePolicy:</span> <span class="string">File</span></span><br><span class="line">      <span class="attr">dnsPolicy:</span> <span class="string">ClusterFirst</span></span><br><span class="line">      <span class="attr">restartPolicy:</span> <span class="string">Always</span></span><br><span class="line">      <span class="attr">schedulerName:</span> <span class="string">default-scheduler</span></span><br><span class="line">      <span class="attr">securityContext:</span> &#123;&#125;</span><br><span class="line">      <span class="attr">terminationGracePeriodSeconds:</span> <span class="number">30</span></span><br><span class="line">  <span class="attr">updateStrategy:</span></span><br><span class="line">    <span class="attr">type:</span> <span class="string">OnDelete</span>	</span><br></pre></td></tr></table></figure>



<h4 id="1-1-6-2-RollingUpdate策略"><a href="#1-1-6-2-RollingUpdate策略" class="headerlink" title="1.1.6.2 RollingUpdate策略"></a>1.1.6.2 RollingUpdate策略</h4><p>RollingUpdate（滚动更新）更新策略会更新一个StatefulSet中所有的Pod，采用与序号索引相反的顺序进行滚动更新。</p>
<ul>
<li>命令行</li>
</ul>
<p>比如Patch一个名称为web的StatefulSet来执行RollingUpdate更新：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl patch statefulset web -p &#39;&#123;&quot;spec&quot;:&#123;&quot;updateStrategy&quot;:&#123;&quot;type&quot;:&quot;RollingUpdate&quot;&#125;&#125;&#125;&#39;</span><br></pre></td></tr></table></figure>

<p>改变容器的镜像进行滚动更新：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl patch statefulset web --type&#x3D;&#39;json&#39; -p&#x3D;&#39;[&#123;&quot;op&quot;: &quot;replace&quot;, &quot;path&quot;: &quot;&#x2F;spec&#x2F;template&#x2F;spec&#x2F;containers&#x2F;0&#x2F;image&quot;, &quot;value&quot;:&quot;dotbalo&#x2F;canary:v1&quot;&#125;]&#39;</span><br></pre></td></tr></table></figure>

<p>如上所述，StatefulSet里的Pod采用和序号相反的顺序更新。在更新下一个Pod前，StatefulSet控制器会终止每一个Pod并等待它们变成Running和Ready状态。在当前顺序变成Running和Ready状态之前，StatefulSet控制器不会更新下一个Pod，但它仍然会重建任何在更新过程中发生故障的Pod，使用它们当前的版本。已经接收到请求的Pod将会被恢复为更新的版本，没有收到请求的Pod则会被恢复为之前的版本</p>
<ul>
<li><p>yaml文件中定义</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">apps/v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">StatefulSet</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">generation:</span> <span class="number">5</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">web</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">default</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">podManagementPolicy:</span> <span class="string">OrderedReady</span></span><br><span class="line">  <span class="attr">replicas:</span> <span class="number">3</span></span><br><span class="line">  <span class="attr">revisionHistoryLimit:</span> <span class="number">10</span></span><br><span class="line">  <span class="attr">selector:</span></span><br><span class="line">    <span class="attr">matchLabels:</span></span><br><span class="line">      <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line">  <span class="attr">serviceName:</span> <span class="string">nginx</span></span><br><span class="line">  <span class="attr">template:</span></span><br><span class="line">    <span class="attr">metadata:</span></span><br><span class="line">      <span class="attr">creationTimestamp:</span> <span class="literal">null</span></span><br><span class="line">      <span class="attr">labels:</span></span><br><span class="line">        <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line">    <span class="attr">spec:</span></span><br><span class="line">      <span class="attr">containers:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">image:</span> <span class="string">nginx:1.15.2</span></span><br><span class="line">        <span class="attr">imagePullPolicy:</span> <span class="string">IfNotPresent</span></span><br><span class="line">        <span class="attr">name:</span> <span class="string">nginx</span></span><br><span class="line">        <span class="attr">ports:</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">containerPort:</span> <span class="number">80</span></span><br><span class="line">          <span class="attr">name:</span> <span class="string">web</span></span><br><span class="line">          <span class="attr">protocol:</span> <span class="string">TCP</span></span><br><span class="line">        <span class="attr">resources:</span> &#123;&#125;</span><br><span class="line">        <span class="attr">terminationMessagePath:</span> <span class="string">/dev/termination-log</span></span><br><span class="line">        <span class="attr">terminationMessagePolicy:</span> <span class="string">File</span></span><br><span class="line">      <span class="attr">dnsPolicy:</span> <span class="string">ClusterFirst</span></span><br><span class="line">      <span class="attr">restartPolicy:</span> <span class="string">Always</span></span><br><span class="line">      <span class="attr">schedulerName:</span> <span class="string">default-scheduler</span></span><br><span class="line">      <span class="attr">securityContext:</span> &#123;&#125;</span><br><span class="line">      <span class="attr">terminationGracePeriodSeconds:</span> <span class="number">30</span></span><br><span class="line">  <span class="attr">updateStrategy:</span></span><br><span class="line">    <span class="attr">rollingUpdate:</span></span><br><span class="line">      <span class="attr">partition:</span> <span class="number">0</span></span><br><span class="line">    <span class="attr">type:</span> <span class="string">RollingUpdate</span>	</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>在更新过程中可以使用 kubectl rollout status sts/<name> 来查看滚动更新的状态：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl rollout status sts&#x2F;web</span><br></pre></td></tr></table></figure>





</li>
</ul>
<h3 id="1-1-7-分段更新（灰度发布）"><a href="#1-1-7-分段更新（灰度发布）" class="headerlink" title="1.1.7 分段更新（灰度发布）"></a>1.1.7 分段更新（灰度发布）</h3><p>StatefulSet可以使用RollingUpdate更新策略的partition参数来分段更新一个StatefulSet。分段更新将会使StatefulSet中其余的所有Pod（序号小于分区）保持当前版本，<strong>只更新序号大于等于分区的Pod</strong>，利用此特性可以简单实现金丝雀发布（灰度发布）或者分阶段推出新功能等。</p>
<blockquote>
<p>注：金丝雀发布是指在黑与白之间能够平滑过渡的一种发布方式。</p>
</blockquote>
<p>比如,以下yaml中partition为0，就表示不更新小于0的pod，换言之就是全部更新：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: apps&#x2F;v1</span><br><span class="line">kind: StatefulSet</span><br><span class="line">metadata:</span><br><span class="line">  generation: 5</span><br><span class="line">  name: web</span><br><span class="line">  namespace: default</span><br><span class="line">spec:</span><br><span class="line">  podManagementPolicy: OrderedReady</span><br><span class="line">  replicas: 3</span><br><span class="line">  revisionHistoryLimit: 10</span><br><span class="line">  selector:</span><br><span class="line">    matchLabels:</span><br><span class="line">      app: nginx</span><br><span class="line">  serviceName: nginx</span><br><span class="line">  template:</span><br><span class="line">    metadata:</span><br><span class="line">      creationTimestamp: null</span><br><span class="line">      labels:</span><br><span class="line">        app: nginx</span><br><span class="line">    spec:</span><br><span class="line">      containers:</span><br><span class="line">      - image: nginx:1.15.2</span><br><span class="line">        imagePullPolicy: IfNotPresent</span><br><span class="line">        name: nginx</span><br><span class="line">        ports:</span><br><span class="line">        - containerPort: 80</span><br><span class="line">          name: web</span><br><span class="line">          protocol: TCP</span><br><span class="line">        resources: &#123;&#125;</span><br><span class="line">        terminationMessagePath: &#x2F;dev&#x2F;termination-log</span><br><span class="line">        terminationMessagePolicy: File</span><br><span class="line">      dnsPolicy: ClusterFirst</span><br><span class="line">      restartPolicy: Always</span><br><span class="line">      schedulerName: default-scheduler</span><br><span class="line">      securityContext: &#123;&#125;</span><br><span class="line">      terminationGracePeriodSeconds: 30</span><br><span class="line">  updateStrategy:</span><br><span class="line">    rollingUpdate:</span><br><span class="line">      partition: 0</span><br><span class="line">    type: RollingUpdate	</span><br></pre></td></tr></table></figure>



<h3 id="1-1-8-级联删除和非级联删除"><a href="#1-1-8-级联删除和非级联删除" class="headerlink" title="1.1.8 级联删除和非级联删除"></a>1.1.8 级联删除和非级联删除</h3><p>删除StatefulSet有两种方式，即级联删除和非级联删除。使用非级联方式删除StatefulSet时，StatefulSet的Pod不会被删除；使用级联删除时，StatefulSet和它的Pod都会被删除。</p>
<h4 id="1-1-9-1-非级联删除"><a href="#1-1-9-1-非级联删除" class="headerlink" title="1.1.9.1 非级联删除"></a>1.1.9.1 非级联删除</h4><ul>
<li><p>使用kubectl delete sts xxx删除StatefulSet时，只需提供–cascade=false参数，就会采用非级联删除，此时删除StatefulSet不会删除它的Pod：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl delete statefulset web --cascade&#x3D;false</span><br></pre></td></tr></table></figure>

<p>查看pod</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"># root @ k8s-master01 in ~ [22:35:14] </span><br><span class="line">$ kubectl get po                                </span><br><span class="line">NAME      READY   STATUS    RESTARTS   AGE</span><br><span class="line">busybox   1&#x2F;1     Running   0          5d</span><br><span class="line">web-0     1&#x2F;1     Running   0          5d</span><br><span class="line">web-1     1&#x2F;1     Running   0          5d</span><br><span class="line">web-2     1&#x2F;1     Running   0          7m6s</span><br><span class="line">web-3     1&#x2F;1     Running   0          7m59s</span><br></pre></td></tr></table></figure>

<blockquote>
<p>可以看到非级联删除StatefulSet后，pod没有被一起删除</p>
</blockquote>
</li>
<li><p>尝试手动删除pod</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl delete po web-2 web-3</span><br></pre></td></tr></table></figure>

<p>查看pod</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"># root @ k8s-master01 in ~ [22:36:50] </span><br><span class="line">$ kubectl get po               </span><br><span class="line">NAME      READY   STATUS    RESTARTS   AGE</span><br><span class="line">busybox   1&#x2F;1     Running   0          5d</span><br><span class="line">web-0     1&#x2F;1     Running   0          5d</span><br><span class="line">web-1     1&#x2F;1     Running   0          5d</span><br></pre></td></tr></table></figure>

<blockquote>
<p>在删除StatefulSet后删除pod，pod不会被重建</p>
</blockquote>
</li>
</ul>
<h4 id="1-1-9-2-级联删除"><a href="#1-1-9-2-级联删除" class="headerlink" title="1.1.9.2 级联删除"></a>1.1.9.2 级联删除</h4><ul>
<li><p>省略–cascade=false参数即为级联删除</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl delete statefulset web</span><br></pre></td></tr></table></figure>

<p>查看</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"># root @ k8s-master01 in ~&#x2F;k8s-test-yaml&#x2F;statefulset [22:39:49] </span><br><span class="line">$ kubectl get po</span><br><span class="line">NAME      READY   STATUS    RESTARTS   AGE</span><br><span class="line">busybox   1&#x2F;1     Running   0          5d</span><br></pre></td></tr></table></figure>

<blockquote>
<p>在级联删除StatefulSet后，pod也随之一并删除</p>
</blockquote>
</li>
<li><p>也可以使用-f参数直接删除StatefulSet和Service</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl delete -f sts-web.yaml </span><br></pre></td></tr></table></figure>

</li>
</ul>

      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color1">kubernetes</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color2">Statefulset</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/kubernetes//" class="article-tag-list-link color1">kubernetes</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/kubernetes/K8s%E8%B5%84%E6%BA%90%E8%B0%83%E5%BA%A6Statefulset/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-kubernetes/k8s基础Master 和 Node" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/kubernetes/k8s%E5%9F%BA%E7%A1%80Master%20%E5%92%8C%20Node/">1. k8s基础Master 和 Node</a>
    </h1>
  

        
        <a href="/2021/04/28/kubernetes/k8s%E5%9F%BA%E7%A1%80Master%20%E5%92%8C%20Node/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h1 id="1-Master-和-node"><a href="#1-Master-和-node" class="headerlink" title="1 Master 和 node"></a>1 Master 和 node</h1><h2 id="1-1-Master节点：整个集群的控制中枢"><a href="#1-1-Master节点：整个集群的控制中枢" class="headerlink" title="1.1 Master节点：整个集群的控制中枢"></a>1.1 Master节点：整个集群的控制中枢</h2><ul>
<li><p>Kube-APIServer：</p>
<p>它 是整个 Kubernetes 集群的“灵魂”，是信息的汇聚中枢，提供了所有内部和外部的 API 请求操作的唯一入口。同时也负责整个集群的认证、授权、访问控制、服务发现等能力。</p>
<p>用户可以通过命令行工具 kubectl 和 APIServer 进行交互，从而实现对集群中进行各种资源的增删改查等操作。APIServer 跟 BorgMaster 非常类似，会将所有的改动持久到 Etcd 中，同时也保存着一份内存拷贝。</p>
</li>
</ul>
<ul>
<li><p>Controller-Manager：</p>
<p>它负责维护整个 Kubernetes 集群的状态，比如多副本创建、滚动更新等。Kube-controller-manager 并不是一个单一组件，内部包含了一组资源控制器，在启动的时候，会通过 goroutine 拉起多个资源控制器。这些控制器的逻辑仅依赖于当前状态，因为在分布式系统中没办法保证全局状态的同步。</p>
<p>同时在实现的时候避免使用过于复杂的状态机，因此每个控制器仅仅对自己对应的资源对象做操作。而且控制器做了很多容错处理，比如增加 retry 机制等。</p>
</li>
<li><p>Scheduler：</p>
<p>它的工作简单来说就是监听未调度的 Pod，按照预定的调度策略绑定到满足条件的节点上。这个工作虽说看起来是三大件中最简单的，但是做的事情可一点不少。</p>
</li>
<li><p>Etcd：</p>
<p>键值数据库，保存一些集群的信息，一般生产环境中建议部署三个以上节点（奇数个）。</p>
</li>
</ul>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20201126215035768.png" alt="image-20201126215035768"></p>
<h2 id="1-2-Node：工作节点"><a href="#1-2-Node：工作节点" class="headerlink" title="1.2  Node：工作节点"></a>1.2  Node：工作节点</h2><p>​    Node又被称为Worker、node节点、minion节点</p>
<ul>
<li><p>容器运行时(可以理解为Docker)主要负责容器的镜像管理以及容器创建及运行。大家都知道的 Docker 就是很常用的容器，此外还有 Kata、Frakti等。只要符合 CRI（Container Runtime Interface，容器运行时接口）规范的运行时，都可以在 Kubernetes 中使用。</p>
</li>
<li><p>Kubelet：负责监听节点上Pod的状态，同时负责上报节点和节点上面Pod的状态，负责与Master节点通信，并管理节点上面的Pod。</p>
</li>
<li><p>Kube-proxy：负责Pod之间的通信和负载均衡，将指定的流量分发到后端正确的机器上。</p>
<p>查看Kube-proxy工作模式：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">curl 127.0.0.1:10249&#x2F;proxyMode</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>Ipvs：监听Master节点增加和删除service以及endpoint的消息，调用Netlink接口创建相应的IPVS规则。通过IPVS规则，将流量转发至相应的Pod上。</p>
</li>
<li><p>Iptables：监听Master节点增加和删除service以及endpoint的消息，对于每一个Service，他都会场景一个iptables规则，将service的clusterIP代理到后端对应的Pod。</p>
</li>
</ul>
<ul>
<li><p>Calico：符合CNI标准的网络插件，给每个Pod生成一个唯一的IP地址，并且把每个节点当做一个路由器。Cilium</p>
</li>
<li><p>CoreDNS：用于Kubernetes集群内部Service的解析，可以让Pod把Service名称解析成IP地址，然后通过Service的IP地址进行连接到对应的应用上。</p>
</li>
</ul>
<h2 id="1-3-Kubernetes端口列表"><a href="#1-3-Kubernetes端口列表" class="headerlink" title="1.3 Kubernetes端口列表"></a>1.3 Kubernetes端口列表</h2><table>
<thead>
<tr>
<th align="left">组件</th>
<th align="left">端口</th>
<th align="left">参数</th>
<th align="left">默认值</th>
<th align="left">协议</th>
<th align="left">必须开启</th>
<th align="left">说明</th>
</tr>
</thead>
<tbody><tr>
<td align="left">kube-apiserver</td>
<td align="left">安全端口</td>
<td align="left">–secure-port</td>
<td align="left">6443</td>
<td align="left">HTTPS</td>
<td align="left">是</td>
<td align="left">-</td>
</tr>
<tr>
<td align="left">kube-apiserver</td>
<td align="left">非安全端口</td>
<td align="left">–insecure-port</td>
<td align="left">8080</td>
<td align="left">HTTP</td>
<td align="left">否，0表示关闭</td>
<td align="left">deprecated</td>
</tr>
<tr>
<td align="left">kubelet</td>
<td align="left">健康检测端口</td>
<td align="left">–healthz-port</td>
<td align="left">10248</td>
<td align="left">HTTP</td>
<td align="left">否，0表示关闭</td>
<td align="left">-</td>
</tr>
<tr>
<td align="left">kube-proxy</td>
<td align="left">指标端口</td>
<td align="left">–metrics-port</td>
<td align="left">10249</td>
<td align="left">HTTP</td>
<td align="left">否，0表示关闭</td>
<td align="left">-</td>
</tr>
<tr>
<td align="left">kubelet</td>
<td align="left">安全端口</td>
<td align="left">–port</td>
<td align="left">10250</td>
<td align="left">HTTPS</td>
<td align="left">是</td>
<td align="left">认证与授权</td>
</tr>
<tr>
<td align="left">kube-scheduler</td>
<td align="left">非安全端口</td>
<td align="left">–insecure-port</td>
<td align="left">10251</td>
<td align="left">HTTP</td>
<td align="left">否，0表示关闭</td>
<td align="left">deprecated</td>
</tr>
<tr>
<td align="left">kube-controller-manager</td>
<td align="left">非安全端口</td>
<td align="left">–insecure-port</td>
<td align="left">10252</td>
<td align="left">HTTP</td>
<td align="left">否，0表示关闭</td>
<td align="left">deprecated</td>
</tr>
<tr>
<td align="left">kubelet</td>
<td align="left">非安全端口</td>
<td align="left">–read-only-port</td>
<td align="left">10255</td>
<td align="left">HTTP</td>
<td align="left">否，0表示关闭</td>
<td align="left">-</td>
</tr>
<tr>
<td align="left">kube-proxy</td>
<td align="left">健康检测端口</td>
<td align="left">–healthz-port</td>
<td align="left">10256</td>
<td align="left">HTTP</td>
<td align="left">否，0表示关闭</td>
<td align="left">-</td>
</tr>
<tr>
<td align="left">kube-controller-manager</td>
<td align="left">安全端口</td>
<td align="left">–secure-port</td>
<td align="left">10257</td>
<td align="left">HTTPS</td>
<td align="left">否，0表示关闭</td>
<td align="left">认证与授权</td>
</tr>
<tr>
<td align="left">kube-scheduler</td>
<td align="left">安全端口</td>
<td align="left">–secure-port</td>
<td align="left">10259</td>
<td align="left">HTTPS</td>
<td align="left">否，0表示关闭</td>
<td align="left">认证与授权</td>
</tr>
</tbody></table>
<h3 id="1-3-1-配置建议"><a href="#1-3-1-配置建议" class="headerlink" title="1.3.1 配置建议"></a>1.3.1 配置建议</h3><p>除了四个安全端口以外，其他端口都建议关闭。另外，kube-apiserver、kube-controller-manager、kube-scheduler的非安全端口这个参数在未来的版本中会被废弃掉。</p>
<h3 id="1-3-2-10257与10259端口"><a href="#1-3-2-10257与10259端口" class="headerlink" title="1.3.2 10257与10259端口"></a>1.3.2 10257与10259端口</h3><p>这两个端口是controller-manager与scheduler的安全端口。kubeadm安装的集群，kube-controller-manager与kube-scheduler的安全端口对应的监听地址为127.0.0.1，开启这个端口的作用主要是用来检查它们健康状态。比如说，我们执行以下命令来查看k8s组件的状态：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">$ kubectl get cs</span><br><span class="line">NAME                 STATUS    MESSAGE             ERROR</span><br><span class="line">scheduler            Healthy   ok                  </span><br><span class="line">controller-manager   Healthy   ok                  </span><br><span class="line">etcd-0               Healthy   &#123;&quot;health&quot;:&quot;true&quot;&#125;</span><br></pre></td></tr></table></figure>

<p>kube-apiserver其实是通过 <code>https://127.0.0.1:10257/healthz</code> 和 <code>https://127.0.0.1:10259/healthz</code> 来判断controller-manager和scheduler的状态。</p>
<p>有时候我们安装了一个N个Master的集群，通过<code>kubectl get cs</code>命令可以看到多个etcd的健康状态，但是却还是只能看到一个scheduler和controller-manager，如下：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">$ kubectl get cs</span><br><span class="line">NAME                 STATUS    MESSAGE             ERROR</span><br><span class="line">scheduler            Healthy   ok                  </span><br><span class="line">controller-manager   Healthy   ok                  </span><br><span class="line">etcd-0               Healthy   &#123;&quot;health&quot;:&quot;true&quot;&#125;</span><br><span class="line">etcd-1               Healthy   &#123;&quot;health&quot;:&quot;true&quot;&#125;</span><br><span class="line">etcd-2               Healthy   &#123;&quot;health&quot;:&quot;true&quot;&#125;</span><br></pre></td></tr></table></figure>

<p>这是因为，kube-apiserver只会检查本节点上的10257和10259端口，不会检查其他Master节点的这两个端口。这也是为什么这两个端口只监听<code>127.0.0.1</code>的原因。</p>

      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color1">kubernetes</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color1">k8s组件</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/kubernetes//" class="article-tag-list-link color1">kubernetes</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/kubernetes/k8s%E5%9F%BA%E7%A1%80Master%20%E5%92%8C%20Node/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-kubernetes/k8s资源调度DaemonSet" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/kubernetes/k8s%E8%B5%84%E6%BA%90%E8%B0%83%E5%BA%A6DaemonSet/">k8s资源调度DaemonSet</a>
    </h1>
  

        
        <a href="/2021/04/28/kubernetes/k8s%E8%B5%84%E6%BA%90%E8%B0%83%E5%BA%A6DaemonSet/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h1 id="1-1-DaemonSet"><a href="#1-1-DaemonSet" class="headerlink" title="1.1  DaemonSet"></a>1.1  DaemonSet</h1><p>DaemonSet（守护进程集）和守护进程类似，它在符合匹配条件的节点上均部署一个Pod。</p>
<h2 id="1-1-1-什么是DaemonSet"><a href="#1-1-1-什么是DaemonSet" class="headerlink" title="1.1.1 什么是DaemonSet"></a>1.1.1 什么是DaemonSet</h2><p>DaemonSet确保全部（或者某些）节点上运行一个Pod副本。当有新节点加入集群时，也会为它们新增一个Pod。当节点从集群中移除时，这些Pod也会被回收，删除DaemonSet将会删除它创建的所有Pod。</p>
<h2 id="1-1-2-使用DaemonSet的一些典型用法："><a href="#1-1-2-使用DaemonSet的一些典型用法：" class="headerlink" title="1.1.2 使用DaemonSet的一些典型用法："></a>1.1.2 使用DaemonSet的一些典型用法：</h2><ul>
<li><p>运行集群存储daemon（守护进程），例如在每个节点上运行Glusterd、Ceph等。</p>
</li>
<li><p>在每个节点运行日志收集daemon，例如Fluentd、Logstash。</p>
</li>
<li><p>在每个节点运行监控daemon，比如Prometheus Node Exporter、Collectd、Datadog代理、New Relic代理或 Ganglia gmond。</p>
</li>
</ul>
<h2 id="1-1-3-编写DaemonSet"><a href="#1-1-3-编写DaemonSet" class="headerlink" title="1.1.3 编写DaemonSet"></a>1.1.3 编写DaemonSet</h2><h3 id="1-1-3-1-创建一个DaemonSet的内容大致如下"><a href="#1-1-3-1-创建一个DaemonSet的内容大致如下" class="headerlink" title="1.1.3.1 创建一个DaemonSet的内容大致如下"></a>1.1.3.1 创建一个DaemonSet的内容大致如下</h3><ul>
<li><p>比如创建一个fluentd的DaemonSet：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">apps/v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">DaemonSet</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">fluentd-es-v2.0.4</span></span><br><span class="line">  <span class="attr">namespace:</span> <span class="string">logging</span></span><br><span class="line">  <span class="attr">labels:</span></span><br><span class="line">    <span class="attr">k8s-app:</span> <span class="string">fluentd-es</span></span><br><span class="line">    <span class="attr">version:</span> <span class="string">v2.0.4</span></span><br><span class="line">    <span class="attr">kubernetes.io/cluster-service:</span> <span class="string">&quot;true&quot;</span></span><br><span class="line">    <span class="attr">addonmanager.kubernetes.io/mode:</span> <span class="string">Reconcile</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">selector:</span></span><br><span class="line">    <span class="attr">matchLabels:</span></span><br><span class="line">      <span class="attr">k8s-app:</span> <span class="string">fluentd-es</span></span><br><span class="line">      <span class="attr">version:</span> <span class="string">v2.0.4</span></span><br><span class="line">  <span class="attr">template:</span></span><br><span class="line">    <span class="attr">metadata:</span></span><br><span class="line">      <span class="attr">labels:</span></span><br><span class="line">        <span class="attr">k8s-app:</span> <span class="string">fluentd-es</span></span><br><span class="line">        <span class="attr">kubernetes.io/cluster-service:</span> <span class="string">&quot;true&quot;</span></span><br><span class="line">        <span class="attr">version:</span> <span class="string">v2.0.4</span></span><br><span class="line">      <span class="comment"># This annotation ensures that fluentd does not get evicted if the node</span></span><br><span class="line">      <span class="comment"># supports critical pod annotation based priority scheme.</span></span><br><span class="line">      <span class="comment"># Note that this does not guarantee admission on the nodes (#40573).</span></span><br><span class="line">      <span class="attr">annotations:</span></span><br><span class="line">        <span class="attr">scheduler.alpha.kubernetes.io/critical-pod:</span> <span class="string">&#x27;&#x27;</span></span><br><span class="line">        <span class="attr">seccomp.security.alpha.kubernetes.io/pod:</span> <span class="string">&#x27;docker/default&#x27;</span></span><br><span class="line">    <span class="attr">spec:</span></span><br><span class="line">      <span class="attr">serviceAccountName:</span> <span class="string">fluentd-es</span></span><br><span class="line">      <span class="attr">containers:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">fluentd-es</span></span><br><span class="line">        <span class="attr">image:</span> <span class="string">k8s.gcr.io/fluentd-elasticsearch:v2.0.4</span></span><br><span class="line">        <span class="attr">env:</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">FLUENTD_ARGS</span></span><br><span class="line">          <span class="attr">value:</span> <span class="string">--no-supervisor</span> <span class="string">-q</span></span><br><span class="line">        <span class="attr">resources:</span></span><br><span class="line">          <span class="attr">limits:</span></span><br><span class="line">            <span class="attr">memory:</span> <span class="string">500Mi</span></span><br><span class="line">          <span class="attr">requests:</span></span><br><span class="line">            <span class="attr">cpu:</span> <span class="string">100m</span></span><br><span class="line">            <span class="attr">memory:</span> <span class="string">200Mi</span></span><br><span class="line">        <span class="attr">volumeMounts:</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">varlog</span></span><br><span class="line">          <span class="attr">mountPath:</span> <span class="string">/var/log</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">varlibdockercontainers</span></span><br><span class="line">          <span class="attr">mountPath:</span> <span class="string">/var/lib/docker/containers</span></span><br><span class="line">          <span class="attr">readOnly:</span> <span class="literal">true</span></span><br><span class="line">        <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">config-volume</span></span><br><span class="line">          <span class="attr">mountPath:</span> <span class="string">/etc/fluent/config.d</span></span><br><span class="line">      <span class="attr">nodeSelector:</span></span><br><span class="line">        <span class="attr">beta.kubernetes.io/fluentd-ds-ready:</span> <span class="string">&quot;true&quot;</span></span><br><span class="line">      <span class="attr">terminationGracePeriodSeconds:</span> <span class="number">30</span></span><br><span class="line">      <span class="attr">volumes:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">varlog</span></span><br><span class="line">        <span class="attr">hostPath:</span></span><br><span class="line">          <span class="attr">path:</span> <span class="string">/var/log</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">varlibdockercontainers</span></span><br><span class="line">        <span class="attr">hostPath:</span></span><br><span class="line">          <span class="attr">path:</span> <span class="string">/var/lib/docker/containers</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">config-volume</span></span><br><span class="line">        <span class="attr">configMap:</span></span><br><span class="line">          <span class="attr">name:</span> <span class="string">fluentd-es-config-v0.1.4</span></span><br></pre></td></tr></table></figure>



</li>
</ul>
<h3 id="1-1-3-2-字段讲解"><a href="#1-1-3-2-字段讲解" class="headerlink" title="1.1.3.2 字段讲解"></a>1.1.3.2 字段讲解</h3><h4 id="1-1-3-2-1-必需字段"><a href="#1-1-3-2-1-必需字段" class="headerlink" title="1.1.3.2.1 必需字段"></a>1.1.3.2.1 必需字段</h4><p>和其他所有Kubernetes配置一样，DaemonSet需要apiVersion、kind和metadata字段，同时也需要一个.spec配置段。</p>
<h4 id="1-1-3-2-2-Pod模板"><a href="#1-1-3-2-2-Pod模板" class="headerlink" title="1.1.3.2.2 Pod模板"></a>1.1.3.2.2 Pod模板</h4><ul>
<li><p>.spec唯一需要的字段是.spec.template。</p>
</li>
<li><p>.spec.template是一个Pod模板，它与Pod具有相同的配置方式，但它不具有apiVersion和kind字段。</p>
</li>
<li><p>除了Pod必需的字段外，在DaemonSet中的Pod模板必须指定合理的标签。</p>
</li>
<li><p>在DaemonSet中的Pod模板必须具有一个RestartPolicy，默认为Always。</p>
</li>
</ul>
<h4 id="1-1-3-2-3-Pod-Selector"><a href="#1-1-3-2-3-Pod-Selector" class="headerlink" title="1.1.3.2.3 Pod Selector"></a>1.1.3.2.3 Pod Selector</h4><ul>
<li><p>.spec.selector字段表示Pod Selector，它与其他资源的.spec.selector的作用相同。</p>
</li>
<li><p>.spec.selector表示一个对象，它由如下两个字段组成：</p>
<ol>
<li><p>matchLabels，与ReplicationController的.spec.selector的作用相同，用于匹配符合条件的Pod。</p>
</li>
<li><p>matchExpressions，允许构建更加复杂的Selector，可以通过指定key、value列表以及与key和value列表相关的操作符。</p>
<blockquote>
<p>如果上述两个字段都指定时，结果表示的是AND关系（逻辑与的关系）。</p>
</blockquote>
</li>
</ol>
</li>
</ul>
<p>.spec.selector必须与.spec.template.metadata.labels相匹配。如果没有指定，默认是等价的，如果它们的配置不匹配，则会被API拒绝。</p>
<h2 id="1-1-4DaemonSet使用"><a href="#1-1-4DaemonSet使用" class="headerlink" title="1.1.4DaemonSet使用"></a>1.1.4DaemonSet使用</h2><h3 id="1-1-4-1-创建一个DaemonSet"><a href="#1-1-4-1-创建一个DaemonSet" class="headerlink" title="1.1.4.1 创建一个DaemonSet"></a>1.1.4.1 创建一个DaemonSet</h3><ul>
<li><p>ds-nginx.yaml文件内容如下：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">apps/v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">DaemonSet</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">labels:</span></span><br><span class="line">    <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">nginx</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">revisionHistoryLimit:</span> <span class="number">10</span></span><br><span class="line">  <span class="attr">selector:</span></span><br><span class="line">    <span class="attr">matchLabels:</span></span><br><span class="line">      <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line">  <span class="attr">template:</span></span><br><span class="line">    <span class="attr">metadata:</span></span><br><span class="line">      <span class="attr">creationTimestamp:</span> <span class="literal">null</span></span><br><span class="line">      <span class="attr">labels:</span></span><br><span class="line">        <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line">    <span class="attr">spec:</span></span><br><span class="line">      <span class="attr">containers:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">image:</span> <span class="string">nginx:1.15.2</span></span><br><span class="line">        <span class="attr">imagePullPolicy:</span> <span class="string">IfNotPresent</span></span><br><span class="line">        <span class="attr">name:</span> <span class="string">nginx</span></span><br><span class="line">        <span class="attr">resources:</span> &#123;&#125;</span><br><span class="line">        <span class="attr">terminationMessagePath:</span> <span class="string">/dev/termination-log</span></span><br><span class="line">        <span class="attr">terminationMessagePolicy:</span> <span class="string">File</span></span><br><span class="line">      <span class="attr">dnsPolicy:</span> <span class="string">ClusterFirst</span></span><br><span class="line">      <span class="attr">restartPolicy:</span> <span class="string">Always</span></span><br><span class="line">      <span class="attr">schedulerName:</span> <span class="string">default-scheduler</span></span><br><span class="line">      <span class="attr">securityContext:</span> &#123;&#125;</span><br><span class="line">      <span class="attr">terminationGracePeriodSeconds:</span> <span class="number">30</span></span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>创建DaemonSet</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f ds-nginx.yaml</span><br></pre></td></tr></table></figure>

<p>查看</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"># root @ k8s-master01 in ~&#x2F;k8s-test-yaml&#x2F;statefulset [23:01:52] </span><br><span class="line">$ kubectl get po</span><br><span class="line">NAME          READY   STATUS    RESTARTS   AGE</span><br><span class="line">nginx-2d85m   1&#x2F;1     Running   0          66s</span><br><span class="line">nginx-84qm4   1&#x2F;1     Running   0          66s</span><br><span class="line">nginx-cfbv6   1&#x2F;1     Running   0          66s</span><br><span class="line">nginx-jsqng   1&#x2F;1     Running   0          66s</span><br><span class="line">nginx-n9wfr   1&#x2F;1     Running   0          66s</span><br></pre></td></tr></table></figure>

<blockquote>
<p>因为没有指定nodeSelector,DaemonSet会在所有节点部署</p>
</blockquote>
</li>
</ul>
<h3 id="1-1-4-2-指定节点nodeSelector部署Pod"><a href="#1-1-4-2-指定节点nodeSelector部署Pod" class="headerlink" title="1.1.4.2 指定节点nodeSelector部署Pod"></a>1.1.4.2 指定节点nodeSelector部署Pod</h3><p>如果指定了.spec.template.spec.nodeSelector，DaemonSet Controller将在与Node Selector（节点选择器）匹配的节点上创建Pod，比如部署在磁盘类型为ssd的节点上（需要提前给节点定义标签Label）：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">containers:</span></span><br><span class="line">  <span class="bullet">-</span> <span class="attr">name:</span> <span class="string">nginx</span></span><br><span class="line">    <span class="attr">image:</span> <span class="string">nginx</span></span><br><span class="line">    <span class="attr">imagePullPolicy:</span> <span class="string">IfNotPresent</span></span><br><span class="line">  <span class="attr">nodeSelector:</span></span><br><span class="line">    <span class="attr">disktype:</span> <span class="string">ssd</span></span><br></pre></td></tr></table></figure>

<blockquote>
<p>Node Selector同样适用于其他Controller。</p>
</blockquote>
<ul>
<li><p>节点定义标签Label</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl label node k8s-node01 k8s-node02 ds&#x3D;true</span><br></pre></td></tr></table></figure>

<p>查看Label</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get node --show-labels</span><br></pre></td></tr></table></figure>
</li>
<li><p>修改上面的ds-nginx.yaml内容如下：</p>
<figure class="highlight yaml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">apiVersion:</span> <span class="string">apps/v1</span></span><br><span class="line"><span class="attr">kind:</span> <span class="string">DaemonSet</span></span><br><span class="line"><span class="attr">metadata:</span></span><br><span class="line">  <span class="attr">labels:</span></span><br><span class="line">    <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line">  <span class="attr">name:</span> <span class="string">nginx</span></span><br><span class="line"><span class="attr">spec:</span></span><br><span class="line">  <span class="attr">revisionHistoryLimit:</span> <span class="number">10</span></span><br><span class="line">  <span class="attr">selector:</span></span><br><span class="line">    <span class="attr">matchLabels:</span></span><br><span class="line">      <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line">  <span class="attr">template:</span></span><br><span class="line">    <span class="attr">metadata:</span></span><br><span class="line">      <span class="attr">creationTimestamp:</span> <span class="literal">null</span></span><br><span class="line">      <span class="attr">labels:</span></span><br><span class="line">        <span class="attr">app:</span> <span class="string">nginx</span></span><br><span class="line">    <span class="attr">spec:</span></span><br><span class="line">      <span class="attr">containers:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">image:</span> <span class="string">nginx:1.15.2</span></span><br><span class="line">        <span class="attr">imagePullPolicy:</span> <span class="string">IfNotPresent</span></span><br><span class="line">        <span class="attr">name:</span> <span class="string">nginx</span></span><br><span class="line">        <span class="attr">resources:</span> &#123;&#125;</span><br><span class="line">        <span class="attr">terminationMessagePath:</span> <span class="string">/dev/termination-log</span></span><br><span class="line">        <span class="attr">terminationMessagePolicy:</span> <span class="string">File</span></span><br><span class="line">      <span class="attr">dnsPolicy:</span> <span class="string">ClusterFirst</span></span><br><span class="line">      <span class="attr">nodeSelector:</span></span><br><span class="line">        <span class="attr">ds:</span> <span class="string">&quot;true&quot;</span></span><br><span class="line">      <span class="attr">restartPolicy:</span> <span class="string">Always</span></span><br><span class="line">      <span class="attr">schedulerName:</span> <span class="string">default-scheduler</span></span><br><span class="line">      <span class="attr">securityContext:</span> &#123;&#125;</span><br><span class="line">      <span class="attr">terminationGracePeriodSeconds:</span> <span class="number">30</span></span><br></pre></td></tr></table></figure>
</li>
<li><p>替换生效</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl replace -f ds-nginx.yaml</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>新增节点只需在相应的节点增加标签即可</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl label node k8s-master01 ds&#x3D;true</span><br></pre></td></tr></table></figure>



</li>
</ul>
<h2 id="1-1-5-DaemonSet的更新和回滚"><a href="#1-1-5-DaemonSet的更新和回滚" class="headerlink" title="1.1.5 DaemonSet的更新和回滚"></a>1.1.5 DaemonSet的更新和回滚</h2><p>如果修改了节点标签（Label），DaemonSet将立刻向新匹配上的节点添加Pod，同时删除不能匹配的节点上的Pod。</p>
<p>在Kubernetes 1.6以后的版本中，可以在DaemonSet上执行滚动更新，未来的Kubernetes版本将支持节点的可控更新。</p>
<p>DaemonSet滚动更新可参考：<a target="_blank" rel="noopener" href="https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/%E3%80%82">https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/。</a></p>
<p>DaemonSet更新策略和StatefulSet类似，也有OnDelete和RollingUpdate两种方式。</p>
<ul>
<li><p>查看DaemonSet更新方式：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get ds&#x2F;nginx-ds -o go-template&#x3D;&#39;&#123;&#123;.spec.updateStrategy.type&#125;&#125;&#123;&#123;&quot;\n&quot;&#125;&#125;&#39;</span><br></pre></td></tr></table></figure>



</li>
</ul>
<h2 id="1-1-6-DaemonSet常用命令行"><a href="#1-1-6-DaemonSet常用命令行" class="headerlink" title="1.1.6 DaemonSet常用命令行"></a>1.1.6 DaemonSet常用命令行</h2><ol>
<li><p>命令式更新</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">kubectl edit ds&#x2F;&lt;daemonset-name&gt;</span><br><span class="line"></span><br><span class="line">kubectl patch ds&#x2F;&lt;daemonset-name&gt; -p&#x3D;&lt;strategic-merge-patch&gt;</span><br></pre></td></tr></table></figure>



</li>
</ol>
<ol start="2">
<li><p>更新镜像</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl set image ds&#x2F;&lt;daemonset-name&gt;&lt;container-name&gt;&#x3D;&lt;container-new-image&gt;--record&#x3D;true</span><br></pre></td></tr></table></figure>



</li>
</ol>
<ol start="3">
<li><p>查看更新状态</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl rollout status ds&#x2F;&lt;daemonset-name&gt;</span><br></pre></td></tr></table></figure>



</li>
</ol>
<ol start="4">
<li>列出所有修订版本<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl rollout history daemonset &lt;daemonset-name&gt;</span><br></pre></td></tr></table></figure>



</li>
</ol>
<ol start="5">
<li>回滚到指定revision<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl rollout undo daemonset &lt;daemonset-name&gt; --to-revision&#x3D;&lt;revision</span><br></pre></td></tr></table></figure></li>
</ol>

      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color1">kubernetes</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color5">DaemonSet</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/kubernetes//" class="article-tag-list-link color1">kubernetes</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/kubernetes/k8s%E8%B5%84%E6%BA%90%E8%B0%83%E5%BA%A6DaemonSet/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-kubernetes/k8s基础Pod" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/kubernetes/k8s%E5%9F%BA%E7%A1%80Pod/">k8s基础Pod</a>
    </h1>
  

        
        <a href="/2021/04/28/kubernetes/k8s%E5%9F%BA%E7%A1%80Pod/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h2 id="1-什么是Pod？"><a href="#1-什么是Pod？" class="headerlink" title="1  什么是Pod？"></a>1  什么是Pod？</h2><p>Pod是Kubernetes中最小的单元</p>
<p>它由一组、一个或多个容器组成</p>
<p>每个Pod还包含了一个Pause容器，Pause容器是Pod的父容器，主要负责僵尸进程的回收管理</p>
<p>通过Pause容器可以使同一个Pod里面的多个容器共享存储、网络、PID、IPC等。</p>
<h2 id="2-定义一个Pod"><a href="#2-定义一个Pod" class="headerlink" title="2  定义一个Pod"></a>2  定义一个Pod</h2><h3 id="2-1-定义一个pod-yaml"><a href="#2-1-定义一个pod-yaml" class="headerlink" title="2.1 定义一个pod.yaml"></a>2.1 定义一个pod.yaml</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: v1 # 必选，API的版本号</span><br><span class="line">kind: Pod       # 必选，类型Pod</span><br><span class="line">metadata:       # 必选，元数据</span><br><span class="line">  name: nginx   # 必选，符合RFC 1035规范的Pod名称</span><br><span class="line">  # namespace: default # 可选，Pod所在的命名空间，不指定默认为default，可以使用-n 指定namespace </span><br><span class="line">  labels:       # 可选，标签选择器，一般用于过滤和区分Pod</span><br><span class="line">    app: nginx</span><br><span class="line">    role: frontend # 可以写多个</span><br><span class="line">  annotations:  # 可选，注释列表，可以写多个</span><br><span class="line">    app: nginx</span><br><span class="line">spec:   # 必选，用于定义容器的详细信息</span><br><span class="line">#  initContainers: # 初始化容器，在容器启动之前执行的一些初始化操作</span><br><span class="line">#  - command:</span><br><span class="line">#    - sh</span><br><span class="line">#    - -c</span><br><span class="line">#    - echo &quot;I am InitContainer for init some configuration&quot;</span><br><span class="line">#    image: busybox</span><br><span class="line">#    imagePullPolicy: IfNotPresent</span><br><span class="line">#    name: init-container</span><br><span class="line">  containers:   # 必选，容器列表</span><br><span class="line">  - name: nginx # 必选，符合RFC 1035规范的容器名称</span><br><span class="line">    image: nginx:1.15.2    # 必选，容器所用的镜像的地址</span><br><span class="line">    imagePullPolicy: IfNotPresent     # 可选，镜像拉取策略, IfNotPresent: 如果宿主机有这个镜像，那就不需要拉取了. Always: 总是拉取, Never: 不管是否存储都不拉去</span><br><span class="line">    command: # 可选，容器启动执行的命令 ENTRYPOINT, arg --&gt; cmd</span><br><span class="line">    - nginx </span><br><span class="line">    - -g</span><br><span class="line">    - &quot;daemon off;&quot;</span><br><span class="line">    workingDir: &#x2F;usr&#x2F;share&#x2F;nginx&#x2F;html       # 可选，容器的工作目录</span><br><span class="line">#    volumeMounts:   # 可选，存储卷配置，可以配置多个</span><br><span class="line">#    - name: webroot # 存储卷名称</span><br><span class="line">#      mountPath: &#x2F;usr&#x2F;share&#x2F;nginx&#x2F;html # 挂载目录</span><br><span class="line">#      readOnly: true        # 只读</span><br><span class="line">    ports:  # 可选，容器需要暴露的端口号列表</span><br><span class="line">    - name: http    # 端口名称</span><br><span class="line">      containerPort: 80     # 端口号</span><br><span class="line">      protocol: TCP # 端口协议，默认TCP</span><br><span class="line">    env:    # 可选，环境变量配置列表</span><br><span class="line">    - name: TZ      # 变量名</span><br><span class="line">      value: Asia&#x2F;Shanghai # 变量的值</span><br><span class="line">    - name: LANG</span><br><span class="line">      value: en_US.utf8</span><br><span class="line">#    resources:      # 可选，资源限制和资源请求限制</span><br><span class="line">#      limits:       # 最大限制设置</span><br><span class="line">#        cpu: 1000m</span><br><span class="line">#        memory: 1024Mi</span><br><span class="line">#      requests:     # 启动所需的资源</span><br><span class="line">#        cpu: 100m</span><br><span class="line">#        memory: 512Mi</span><br><span class="line">#    startupProbe: # 可选，检测容器内进程是否完成启动。注意三种检查方式同时只能使用一种。</span><br><span class="line">#      httpGet:      # httpGet检测方式，生产环境建议使用httpGet实现接口级健康检查，健康检查由应用程序提供。</span><br><span class="line">#            path: &#x2F;api&#x2F;successStart # 检查路径</span><br><span class="line">#            port: 80</span><br><span class="line">#    readinessProbe: # 可选，健康检查。注意三种检查方式同时只能使用一种。</span><br><span class="line">#      httpGet:      # httpGet检测方式，生产环境建议使用httpGet实现接口级健康检查，健康检查由应用程序提供。</span><br><span class="line">#            path: &#x2F; # 检查路径</span><br><span class="line">#            port: 80        # 监控端口</span><br><span class="line">#    livenessProbe:  # 可选，健康检查</span><br><span class="line">      #exec:        # 执行容器命令检测方式</span><br><span class="line">            #command: </span><br><span class="line">            #- cat</span><br><span class="line">            #- &#x2F;health</span><br><span class="line">    #httpGet:       # httpGet检测方式</span><br><span class="line">    #   path: &#x2F;_health # 检查路径</span><br><span class="line">    #   port: 8080</span><br><span class="line">    #   httpHeaders: # 检查的请求头</span><br><span class="line">    #   - name: end-user</span><br><span class="line">    #     value: Jason </span><br><span class="line">#      tcpSocket:    # 端口检测方式</span><br><span class="line">#            port: 80</span><br><span class="line">#      initialDelaySeconds: 60       # 初始化时间</span><br><span class="line">#      timeoutSeconds: 2     # 超时时间</span><br><span class="line">#      periodSeconds: 5      # 检测间隔</span><br><span class="line">#      successThreshold: 1 # 检查成功为2次表示就绪</span><br><span class="line">#      failureThreshold: 2 # 检测失败1次表示未就绪</span><br><span class="line">#    lifecycle:</span><br><span class="line">#      postStart: # 容器创建完成后执行的指令, 可以是exec httpGet TCPSocket</span><br><span class="line">#        exec:</span><br><span class="line">#          command:</span><br><span class="line">#          - sh</span><br><span class="line">#          - -c</span><br><span class="line">#          - &#39;mkdir &#x2F;data&#x2F; &#39;</span><br><span class="line">#      preStop:</span><br><span class="line">#        httpGet:      </span><br><span class="line">#              path: &#x2F;</span><br><span class="line">#              port: 80</span><br><span class="line">      #  exec:</span><br><span class="line">      #    command:</span><br><span class="line">      #    - sh</span><br><span class="line">      #    - -c</span><br><span class="line">      #    - sleep 9</span><br><span class="line">  restartPolicy: Always   # 可选，默认为Always，容器故障或者没有启动成功，那就自动该容器，Onfailure: 容器以不为0的状态终止，自动重启该容器, Never:无论何种状态，都不会重启</span><br><span class="line">  #nodeSelector: # 可选，指定Node节点</span><br><span class="line">  #      region: subnet7</span><br><span class="line">#  imagePullSecrets:     # 可选，拉取镜像使用的secret，可以配置多个</span><br><span class="line">#  - name: default-dockercfg-86258</span><br><span class="line">#  hostNetwork: false    # 可选，是否为主机模式，如是，会占用主机端口</span><br><span class="line">#  volumes:      # 共享存储卷列表</span><br><span class="line">#  - name: webroot # 名称，与上述对应</span><br><span class="line">#    emptyDir: &#123;&#125;    # 挂载目录</span><br><span class="line">#        #hostPath:              # 挂载本机目录</span><br><span class="line">#        #  path: &#x2F;etc&#x2F;hosts</span><br><span class="line"></span><br></pre></td></tr></table></figure>



<h3 id="2-2-创建pod"><a href="#2-2-创建pod" class="headerlink" title="2.2 创建pod"></a>2.2 创建pod</h3><p>同一个namespace下不能有同名的pod</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f pod.yaml</span><br></pre></td></tr></table></figure>

<p>或</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl apply -f pod.yaml</span><br></pre></td></tr></table></figure>

<p>pod.yaml修改后可以使用apply重新创建pod</p>
<h3 id="2-3-查看pod状态"><a href="#2-3-查看pod状态" class="headerlink" title="2.3 查看pod状态"></a>2.3 查看pod状态</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get po</span><br></pre></td></tr></table></figure>

<p>或</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get po nginx</span><br></pre></td></tr></table></figure>

<blockquote>
<p>nginx为yaml文件中<code>metdata.name</code>指定的值</p>
</blockquote>
<h3 id="2-4-查看pod标签"><a href="#2-4-查看pod标签" class="headerlink" title="2.4 查看pod标签"></a>2.4 查看pod标签</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get  po --show-labels</span><br></pre></td></tr></table></figure>



<h3 id="2-5-创建namespace和指定namespace"><a href="#2-5-创建namespace和指定namespace" class="headerlink" title="2.5 创建namespace和指定namespace"></a>2.5 创建namespace和指定namespace</h3><ul>
<li><p>创建namespace</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create ns myns</span><br></pre></td></tr></table></figure>
</li>
<li><p>创建pod并指定namespace</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f pod.yaml -n myns</span><br></pre></td></tr></table></figure>



</li>
</ul>
<h3 id="2-6-删除指定pod"><a href="#2-6-删除指定pod" class="headerlink" title="2.6 删除指定pod"></a>2.6 删除指定pod</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl delete po nginx</span><br></pre></td></tr></table></figure>



<h3 id="2-7-查看指定pod详细信息"><a href="#2-7-查看指定pod详细信息" class="headerlink" title="2.7 查看指定pod详细信息"></a>2.7 查看指定pod详细信息</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl describe po nginx</span><br></pre></td></tr></table></figure>



<h2 id="3-Pod探针"><a href="#3-Pod探针" class="headerlink" title="3  Pod探针"></a>3  Pod探针</h2><h3 id="3-1-pod有哪些探针？"><a href="#3-1-pod有哪些探针？" class="headerlink" title="3.1 pod有哪些探针？"></a>3.1 pod有哪些探针？</h3><h4 id="3-1-1-StartupProbe："><a href="#3-1-1-StartupProbe：" class="headerlink" title="3.1.1 StartupProbe："></a>3.1.1 StartupProbe：</h4><ul>
<li>k8s1.16版本后新加的探测方式，用于判断容器内应用程序是否已经启动。如果配置了startupProbe，就会先禁止其他的探测，直到它成功为止，成功后将不在进行探测。</li>
</ul>
<h4 id="3-1-2-LivenessProbe："><a href="#3-1-2-LivenessProbe：" class="headerlink" title="3.1.2 LivenessProbe："></a>3.1.2 LivenessProbe：</h4><ul>
<li>用于探测容器是否运行，如果探测失败，kubelet会根据配置的重启策略进行相应的处理。若没有配置该探针，默认就是success。</li>
</ul>
<h4 id="3-1-3-ReadinessProbe："><a href="#3-1-3-ReadinessProbe：" class="headerlink" title="3.1.3 ReadinessProbe："></a>3.1.3 ReadinessProbe：</h4><ul>
<li>一般用于探测容器内的程序是否健康，它的返回值如果为success，那么久代表这个容器已经完成启动，并且程序已经是可以接受流量的状态。</li>
</ul>
<h3 id=""><a href="#" class="headerlink" title=""></a></h3><h3 id="3-2-为什么要引入StartupProbe"><a href="#3-2-为什么要引入StartupProbe" class="headerlink" title="3.2 为什么要引入StartupProbe"></a>3.2 为什么要引入StartupProbe</h3><p>在早期版本没有引入StartupProbe时，当容器启动较慢时，LivenessProbe检测在容器异常重启时间太久。</p>
<h2 id="4-Pod探针的检测方式"><a href="#4-Pod探针的检测方式" class="headerlink" title="4  Pod探针的检测方式"></a>4  Pod探针的检测方式</h2><ul>
<li><p>ExecAction：在容器内执行一个命令，如果返回值为0，则认为容器健康。</p>
</li>
<li><p>TCPSocketAction：通过TCP连接检查容器内的端口是否是通的，如果是通的就认为容器健康。</p>
</li>
<li><p>HTTPGetAction：通过应用程序暴露的API地址来检查程序是否是正常的，如果状态码为200~400之间，则认为容器健康。</p>
</li>
</ul>
<p>一般推荐使用HTTPGetAction接口级别来检测</p>
<h2 id="5-探针检查参数配置"><a href="#5-探针检查参数配置" class="headerlink" title="5  探针检查参数配置"></a>5  探针检查参数配置</h2><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">initialDelaySeconds: 60    # 初始化时间</span><br><span class="line"></span><br><span class="line">timeoutSeconds: 2   # 超时时间</span><br><span class="line"></span><br><span class="line">periodSeconds: 5   # 检测间隔</span><br><span class="line"></span><br><span class="line">successThreshold: 1 # 检查成功为1次表示就绪</span><br><span class="line"></span><br><span class="line">failureThreshold: 2 # 检测失败2次表示未就绪</span><br></pre></td></tr></table></figure>



<h3 id="5-1-配置注意事项"><a href="#5-1-配置注意事项" class="headerlink" title="5.1 配置注意事项"></a>5.1 配置注意事项</h3><ul>
<li><p>initialDelaySeconds设置时间过长，会导致发布时间过慢</p>
</li>
<li><p>timeoutSeconds一般推荐设置1-2秒</p>
</li>
<li><p>其余参数类似都不能设置过长或过短</p>
</li>
</ul>
<h2 id="6-Prestart和Prestop"><a href="#6-Prestart和Prestop" class="headerlink" title="6 Prestart和Prestop"></a>6 Prestart和Prestop</h2><h2 id="6-1-Prestart"><a href="#6-1-Prestart" class="headerlink" title="6.1 Prestart"></a>6.1 Prestart</h2><p>容器创建完成后运行的命令，可以说exec，tcpSocket，httpGet。Prestart并不能保证每次执行都在CMD指令前运行</p>
<h2 id="6-2-Prestop"><a href="#6-2-Prestop" class="headerlink" title="6.2 Prestop"></a>6.2 Prestop</h2><h3 id="6-2-1-用户推出流程"><a href="#6-2-1-用户推出流程" class="headerlink" title="6.2.1 用户推出流程"></a>6.2.1 用户推出流程</h3><ul>
<li>用户执行删除操作<ol>
<li>执行Prestop指令</li>
<li>EndPoint删除该Pod的IP地址</li>
<li>Pod状态变更为Terminating</li>
</ol>
</li>
</ul>
<h3 id="6-2-2-以springcloud为例"><a href="#6-2-2-以springcloud为例" class="headerlink" title="6.2.2 以springcloud为例"></a>6.2.2 以springcloud为例</h3><p>先去请求eureka接口，把自己的IP地址和端口号，进行下线，eureka从注册表中删除该应用的IP地址。然后容器进行Perstop设置的 sleep 90；kill <code>pgrep java</code></p>
 <figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">preStop:</span><br><span class="line">  exec:</span><br><span class="line">    command:</span><br><span class="line">    - sh</span><br><span class="line">    - -c</span><br><span class="line">    - sleep 90</span><br></pre></td></tr></table></figure>



<blockquote>
<p>Perstop中像 sleep命令的最终生效时间以  terminationGracePeriodSeconds 设定的值为准</p>
</blockquote>

      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color1">kubernetes</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color4">Pod</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/kubernetes//" class="article-tag-list-link color1">kubernetes</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/kubernetes/k8s%E5%9F%BA%E7%A1%80Pod/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
    <article id="post-kubernetes/k8s资源调度HPA" class="article article-type-post  article-index" itemscope itemprop="blogPost">
  <div class="article-inner">
    
      <header class="article-header">
        
  
    <h1 itemprop="name">
      <a class="article-title" href="/2021/04/28/kubernetes/k8s%E8%B5%84%E6%BA%90%E8%B0%83%E5%BA%A6HPA/">k8s资源调度HPA</a>
    </h1>
  

        
        <a href="/2021/04/28/kubernetes/k8s%E8%B5%84%E6%BA%90%E8%B0%83%E5%BA%A6HPA/" class="archive-article-date">
  	<time datetime="2021-04-28T00:25:00.000Z" itemprop="datePublished"><i class="icon-calendar icon"></i>2021-04-28</time>
</a>
        
      </header>
    
    <div class="article-entry" itemprop="articleBody">
      
        <h1 id="1-HPA"><a href="#1-HPA" class="headerlink" title="1. HPA"></a>1. HPA</h1><h2 id="1-1-什么是HPA？"><a href="#1-1-什么是HPA？" class="headerlink" title="1.1 什么是HPA？"></a>1.1 什么是HPA？</h2><p>​    HPA（Horizontal Pod Autoscaler，水平Pod自动伸缩器）可根据观察到的CPU、内存使用率或自定义度量标准来自动扩展或缩容Pod的数量。</p>
<p>​    <strong>HPA不适用于无法缩放的对象，比如DaemonSet。</strong></p>
<p>​    HPA控制器会定期调整RC或Deployment的副本数，以使观察到的平均CPU利用率与用户指定的目标相匹配。</p>
<p>​    HPA需要metrics-server（项目地址：<a target="_blank" rel="noopener" href="https://github.com/kubernetes-incubator/metrics-server%EF%BC%89%E8%8E%B7%E5%8F%96%E5%BA%A6%E9%87%8F%E6%8C%87%E6%A0%87%EF%BC%8C%E7%94%B1%E4%BA%8E%E5%9C%A8%E9%AB%98%E5%8F%AF%E7%94%A8%E9%9B%86%E7%BE%A4%E5%AE%89%E8%A3%85%E4%B8%AD%E5%B7%B2%E7%BB%8F%E5%AE%89%E8%A3%85%E4%BA%86metrics-server%EF%BC%8C%E6%89%80%E4%BB%A5%E6%9C%AC%E8%8A%82%E7%9A%84%E5%AE%9E%E8%B7%B5%E9%83%A8%E5%88%86%E6%97%A0%E9%A1%BB%E5%86%8D%E6%AC%A1%E5%AE%89%E8%A3%85%E3%80%82">https://github.com/kubernetes-incubator/metrics-server）获取度量指标，由于在高可用集群安装中已经安装了metrics-server，所以本节的实践部分无须再次安装。</a></p>
<p>​    使用HPA必须定义 Requests参数</p>
<h2 id="1-2-HPA实践"><a href="#1-2-HPA实践" class="headerlink" title="1.2 HPA实践"></a>1.2 HPA实践</h2><p>在生产环境中，总会有一些意想不到的事情发生，比如公司网站流量突然升高，此时之前创建的Pod已不足以撑住所有的访问，而运维人员也不可能24小时守着业务服务，这时就可以通过配置HPA，实现负载过高的情况下自动扩容Pod副本数以分摊高并发的流量，当流量恢复正常后，HPA会自动缩减Pod的数量。</p>
<ul>
<li><p>定义一个nginx-deploy.yaml：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br></pre></td><td class="code"><pre><span class="line">apiVersion: apps&#x2F;v1</span><br><span class="line">kind: Deployment</span><br><span class="line">metadata:</span><br><span class="line">  annotations:</span><br><span class="line">    deployment.kubernetes.io&#x2F;revision: &quot;12&quot;</span><br><span class="line">    kubernetes.io&#x2F;change-cause: kubectl set image deploy nginx nginx&#x3D;nginx:1.15.3</span><br><span class="line">      --record&#x3D;true</span><br><span class="line">  generation: 19</span><br><span class="line">  labels:</span><br><span class="line">    app: nginx</span><br><span class="line">  name: nginx</span><br><span class="line">  namespace: default</span><br><span class="line">spec:</span><br><span class="line">  progressDeadlineSeconds: 600</span><br><span class="line">  replicas: 2</span><br><span class="line">  revisionHistoryLimit: 10</span><br><span class="line">  selector:</span><br><span class="line">    matchLabels:</span><br><span class="line">      app: nginx</span><br><span class="line">  strategy:</span><br><span class="line">    rollingUpdate:</span><br><span class="line">      maxSurge: 25%</span><br><span class="line">      maxUnavailable: 25%</span><br><span class="line">    type: RollingUpdate</span><br><span class="line">  template:</span><br><span class="line">    metadata:</span><br><span class="line">      creationTimestamp: null</span><br><span class="line">      labels:</span><br><span class="line">        app: nginx</span><br><span class="line">    spec:</span><br><span class="line">      containers:</span><br><span class="line">      - image: nginx:1.15.3</span><br><span class="line">        imagePullPolicy: IfNotPresent</span><br><span class="line">        name: nginx</span><br><span class="line">        resources:</span><br><span class="line">          limits:</span><br><span class="line">            cpu: 200m</span><br><span class="line">            memory: 128Mi</span><br><span class="line">          requests:</span><br><span class="line">            cpu: 10m</span><br><span class="line">            memory: 16Mi</span><br><span class="line">        terminationMessagePath: &#x2F;dev&#x2F;termination-log</span><br><span class="line">        terminationMessagePolicy: File</span><br><span class="line">      dnsPolicy: ClusterFirst</span><br><span class="line">      restartPolicy: Always</span><br><span class="line">      schedulerName: default-scheduler</span><br><span class="line">      securityContext: &#123;&#125;</span><br><span class="line">      terminationGracePeriodSeconds: 30</span><br></pre></td></tr></table></figure>

</li>
</ul>
<blockquote>
<p>必须要定义requests，不然无法使用HPA</p>
<p>其中：</p>
<p>​          cpu: 10m  单位后缀m表示“千分之一个核心”，所以这个资源对象定义了容器进程需要10/1000的核心（1%）</p>
</blockquote>
<ul>
<li><p>创建deploy</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl create -f nginx-deploy.yaml</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>临时开启nginx的端口，实际使用时需要定义service：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl expose deployment nginx --port&#x3D;80</span><br></pre></td></tr></table></figure>
</li>
<li><p>使用kubectl autoscale创建HPA：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl autoscale deployment nginx --cpu-percent&#x3D;10 --min&#x3D;1 --max&#x3D;10</span><br></pre></td></tr></table></figure>

<p>此HPA将根据CPU的使用率自动增加和减少副本数量，上述设置的是</p>
<ol>
<li>CPU使用率超过10%（–cpu-percent参数指定）即会增加Pod的数量，以保持所有Pod的平均CPU利用率为10%</li>
<li>允许最大的Pod数量为10（–max），最少的Pod数为1（–min）</li>
</ol>
</li>
</ul>
<ul>
<li><p>查看当前HPA状态，因为未对其发送任何请求，所以当前CPU使用率为0%：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get hpa </span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>查看当前Nginx的Service地址</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get service</span><br></pre></td></tr></table></figure>



</li>
</ul>
<ul>
<li><p>测试增加负债：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">while true; do curl http:&#x2F;&#x2F;10.108.168.182 &gt; &#x2F;dev&#x2F;null; done</span><br></pre></td></tr></table></figure>

<p>可以在多个终端运行提升负债</p>
</li>
<li><p>1分钟左右再次查看HPA：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">kubectl get hpa </span><br></pre></td></tr></table></figure>

<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20201216223739977.png" alt="image-20201216223739977"></p>
</li>
</ul>
<blockquote>
<p>负债上升并且超过设定的百分比</p>
</blockquote>
<ul>
<li><p>再次查看Pod，可以看到nginx-server的Pod已经在扩容阶段：</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20201216223913634.png" alt="image-20201216223913634"></p>
</li>
</ul>
<ul>
<li><p>在增加负荷的终端，按Ctrl+C键终止访问。</p>
<p>停止5分钟后再次查看HPA和deployment，此时副本已经恢复为1：</p>
<p><img src="https://gitee.com/cxsw_zyj/my_picture/raw/master/python/image-20201216224625746.png" alt="image-20201216224625746"></p>
</li>
</ul>

      

      
    </div>
    <div class="article-info article-info-index">
      
      
	<div class="article-tag tagcloud">
		<i class="icon-price-tags icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color3">笔记</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color1">kubernetes</a>
        		</li>
      		 
        		<li class="article-tag-list-item">
        			<a href="javascript:void(0)" class="js-tag article-tag-list-link color4">HPA</a>
        		</li>
      		
		</ul>
	</div>

      
	<div class="article-category tagcloud">
		<i class="icon-book icon"></i>
		<ul class="article-tag-list">
			 
        		<li class="article-tag-list-item">
        			<a href="/categories/kubernetes//" class="article-tag-list-link color1">kubernetes</a>
        		</li>
      		
		</ul>
	</div>


      
        <p class="article-more-link">
          <a class="article-more-a" href="/2021/04/28/kubernetes/k8s%E8%B5%84%E6%BA%90%E8%B0%83%E5%BA%A6HPA/">展开全文 >></a>
        </p>
      
        <!-- 《添加版权声明 -->
        
            <!-- 《添加版权声明 -->
<!--添加版权声明https://github.com/JoeyBling/hexo-theme-yilia-plus/commit/c1215e132f6d5621c5fea83d3c4f7ccbcca074a3-->


<!-- #版权基础设定：0-关闭声明； 1-文章对应的md文件里有declare: true属性，才有版权声明； 2-所有文章均有版权声明 -->

  <div class="declare" hidden="hidden"></div>

<!-- 添加版权声明》 -->
        
        <!-- 添加版权声明》 -->
      
      <div class="clearfix"></div>
    </div>
  </div>
</article>

<aside class="wrap-side-operation">
    <div class="mod-side-operation">
        
        <div class="jump-container" id="js-jump-container" style="display:none;">
            <a href="javascript:void(0)" class="mod-side-operation__jump-to-top">
                <i class="icon-font icon-back"></i>
            </a>
            <div id="js-jump-plan-container" class="jump-plan-container" style="top: -11px;">
                <i class="icon-font icon-plane jump-plane"></i>
            </div>
        </div>
        
        
    </div>
</aside>




  
  
    <nav id="page-nav">
      <span class="page-number current">1</span><a class="page-number" href="/page/2/">2</a><a class="page-number" href="/page/3/">3</a><span class="space">&hellip;</span><a class="page-number" href="/page/6/">6</a><a class="extend next" rel="next" href="/page/2/">Next &amp;raquo;</a>
    </nav>
  


          </div>
        </div>
      </div>
      <footer id="footer">
  <div class="outer">
    <div id="footer-info">
    	<div class="footer-left">
    		&copy; 2021 宸轩少翁
    	</div>
      	<div class="footer-right">
      		<a href="http://hexo.io/" target="_blank">Hexo</a>  Theme <a href="https://github.com/litten/hexo-theme-yilia" target="_blank">Yilia</a> by Litten
      	</div>
    </div>
	        <!-- 不蒜子统计 -->            
	   <script async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js">
	   </script>	      
		本站总访问量<span id="busuanzi_value_site_pv"></span>次   
  </div>
</footer>
    </div>
    <script>
	var yiliaConfig = {
		mathjax: false,
		isHome: true,
		isPost: false,
		isArchive: false,
		isTag: false,
		isCategory: false,
		open_in_new: false,
		toc_hide_index: true,
		root: "/",
		innerArchive: true,
		showTags: false
	}
</script>

<script>!function(t){function n(e){if(r[e])return r[e].exports;var i=r[e]={exports:{},id:e,loaded:!1};return t[e].call(i.exports,i,i.exports,n),i.loaded=!0,i.exports}var r={};n.m=t,n.c=r,n.p="./",n(0)}([function(t,n,r){r(195),t.exports=r(191)},function(t,n,r){var e=r(3),i=r(52),o=r(27),u=r(28),c=r(53),f="prototype",a=function(t,n,r){var s,l,h,v,p=t&a.F,d=t&a.G,y=t&a.S,g=t&a.P,b=t&a.B,m=d?e:y?e[n]||(e[n]={}):(e[n]||{})[f],x=d?i:i[n]||(i[n]={}),w=x[f]||(x[f]={});d&&(r=n);for(s in r)l=!p&&m&&void 0!==m[s],h=(l?m:r)[s],v=b&&l?c(h,e):g&&"function"==typeof h?c(Function.call,h):h,m&&u(m,s,h,t&a.U),x[s]!=h&&o(x,s,v),g&&w[s]!=h&&(w[s]=h)};e.core=i,a.F=1,a.G=2,a.S=4,a.P=8,a.B=16,a.W=32,a.U=64,a.R=128,t.exports=a},function(t,n,r){var e=r(6);t.exports=function(t){if(!e(t))throw TypeError(t+" is not an object!");return t}},function(t,n){var r=t.exports="undefined"!=typeof window&&window.Math==Math?window:"undefined"!=typeof self&&self.Math==Math?self:Function("return this")();"number"==typeof __g&&(__g=r)},function(t,n){t.exports=function(t){try{return!!t()}catch(t){return!0}}},function(t,n){var r=t.exports="undefined"!=typeof window&&window.Math==Math?window:"undefined"!=typeof self&&self.Math==Math?self:Function("return this")();"number"==typeof __g&&(__g=r)},function(t,n){t.exports=function(t){return"object"==typeof t?null!==t:"function"==typeof t}},function(t,n,r){var e=r(126)("wks"),i=r(76),o=r(3).Symbol,u="function"==typeof o;(t.exports=function(t){return e[t]||(e[t]=u&&o[t]||(u?o:i)("Symbol."+t))}).store=e},function(t,n){var r={}.hasOwnProperty;t.exports=function(t,n){return r.call(t,n)}},function(t,n,r){var e=r(94),i=r(33);t.exports=function(t){return e(i(t))}},function(t,n,r){t.exports=!r(4)(function(){return 7!=Object.defineProperty({},"a",{get:function(){return 7}}).a})},function(t,n,r){var e=r(2),i=r(167),o=r(50),u=Object.defineProperty;n.f=r(10)?Object.defineProperty:function(t,n,r){if(e(t),n=o(n,!0),e(r),i)try{return u(t,n,r)}catch(t){}if("get"in r||"set"in r)throw TypeError("Accessors not supported!");return"value"in r&&(t[n]=r.value),t}},function(t,n,r){t.exports=!r(18)(function(){return 7!=Object.defineProperty({},"a",{get:function(){return 7}}).a})},function(t,n,r){var e=r(14),i=r(22);t.exports=r(12)?function(t,n,r){return e.f(t,n,i(1,r))}:function(t,n,r){return t[n]=r,t}},function(t,n,r){var e=r(20),i=r(58),o=r(42),u=Object.defineProperty;n.f=r(12)?Object.defineProperty:function(t,n,r){if(e(t),n=o(n,!0),e(r),i)try{return u(t,n,r)}catch(t){}if("get"in r||"set"in r)throw TypeError("Accessors not supported!");return"value"in r&&(t[n]=r.value),t}},function(t,n,r){var e=r(40)("wks"),i=r(23),o=r(5).Symbol,u="function"==typeof o;(t.exports=function(t){return e[t]||(e[t]=u&&o[t]||(u?o:i)("Symbol."+t))}).store=e},function(t,n,r){var e=r(67),i=Math.min;t.exports=function(t){return t>0?i(e(t),9007199254740991):0}},function(t,n,r){var e=r(46);t.exports=function(t){return Object(e(t))}},function(t,n){t.exports=function(t){try{return!!t()}catch(t){return!0}}},function(t,n,r){var e=r(63),i=r(34);t.exports=Object.keys||function(t){return e(t,i)}},function(t,n,r){var e=r(21);t.exports=function(t){if(!e(t))throw TypeError(t+" is not an object!");return t}},function(t,n){t.exports=function(t){return"object"==typeof t?null!==t:"function"==typeof t}},function(t,n){t.exports=function(t,n){return{enumerable:!(1&t),configurable:!(2&t),writable:!(4&t),value:n}}},function(t,n){var r=0,e=Math.random();t.exports=function(t){return"Symbol(".concat(void 0===t?"":t,")_",(++r+e).toString(36))}},function(t,n){var r={}.hasOwnProperty;t.exports=function(t,n){return r.call(t,n)}},function(t,n){var r=t.exports={version:"2.4.0"};"number"==typeof __e&&(__e=r)},function(t,n){t.exports=function(t){if("function"!=typeof t)throw TypeError(t+" is not a function!");return t}},function(t,n,r){var e=r(11),i=r(66);t.exports=r(10)?function(t,n,r){return e.f(t,n,i(1,r))}:function(t,n,r){return t[n]=r,t}},function(t,n,r){var e=r(3),i=r(27),o=r(24),u=r(76)("src"),c="toString",f=Function[c],a=(""+f).split(c);r(52).inspectSource=function(t){return f.call(t)},(t.exports=function(t,n,r,c){var f="function"==typeof r;f&&(o(r,"name")||i(r,"name",n)),t[n]!==r&&(f&&(o(r,u)||i(r,u,t[n]?""+t[n]:a.join(String(n)))),t===e?t[n]=r:c?t[n]?t[n]=r:i(t,n,r):(delete t[n],i(t,n,r)))})(Function.prototype,c,function(){return"function"==typeof this&&this[u]||f.call(this)})},function(t,n,r){var e=r(1),i=r(4),o=r(46),u=function(t,n,r,e){var i=String(o(t)),u="<"+n;return""!==r&&(u+=" "+r+'="'+String(e).replace(/"/g,"&quot;")+'"'),u+">"+i+"</"+n+">"};t.exports=function(t,n){var r={};r[t]=n(u),e(e.P+e.F*i(function(){var n=""[t]('"');return n!==n.toLowerCase()||n.split('"').length>3}),"String",r)}},function(t,n,r){var e=r(115),i=r(46);t.exports=function(t){return e(i(t))}},function(t,n,r){var e=r(116),i=r(66),o=r(30),u=r(50),c=r(24),f=r(167),a=Object.getOwnPropertyDescriptor;n.f=r(10)?a:function(t,n){if(t=o(t),n=u(n,!0),f)try{return a(t,n)}catch(t){}if(c(t,n))return i(!e.f.call(t,n),t[n])}},function(t,n,r){var e=r(24),i=r(17),o=r(145)("IE_PROTO"),u=Object.prototype;t.exports=Object.getPrototypeOf||function(t){return t=i(t),e(t,o)?t[o]:"function"==typeof t.constructor&&t instanceof t.constructor?t.constructor.prototype:t instanceof Object?u:null}},function(t,n){t.exports=function(t){if(void 0==t)throw TypeError("Can't call method on  "+t);return t}},function(t,n){t.exports="constructor,hasOwnProperty,isPrototypeOf,propertyIsEnumerable,toLocaleString,toString,valueOf".split(",")},function(t,n){t.exports={}},function(t,n){t.exports=!0},function(t,n){n.f={}.propertyIsEnumerable},function(t,n,r){var e=r(14).f,i=r(8),o=r(15)("toStringTag");t.exports=function(t,n,r){t&&!i(t=r?t:t.prototype,o)&&e(t,o,{configurable:!0,value:n})}},function(t,n,r){var e=r(40)("keys"),i=r(23);t.exports=function(t){return e[t]||(e[t]=i(t))}},function(t,n,r){var e=r(5),i="__core-js_shared__",o=e[i]||(e[i]={});t.exports=function(t){return o[t]||(o[t]={})}},function(t,n){var r=Math.ceil,e=Math.floor;t.exports=function(t){return isNaN(t=+t)?0:(t>0?e:r)(t)}},function(t,n,r){var e=r(21);t.exports=function(t,n){if(!e(t))return t;var r,i;if(n&&"function"==typeof(r=t.toString)&&!e(i=r.call(t)))return i;if("function"==typeof(r=t.valueOf)&&!e(i=r.call(t)))return i;if(!n&&"function"==typeof(r=t.toString)&&!e(i=r.call(t)))return i;throw TypeError("Can't convert object to primitive value")}},function(t,n,r){var e=r(5),i=r(25),o=r(36),u=r(44),c=r(14).f;t.exports=function(t){var n=i.Symbol||(i.Symbol=o?{}:e.Symbol||{});"_"==t.charAt(0)||t in n||c(n,t,{value:u.f(t)})}},function(t,n,r){n.f=r(15)},function(t,n){var r={}.toString;t.exports=function(t){return r.call(t).slice(8,-1)}},function(t,n){t.exports=function(t){if(void 0==t)throw TypeError("Can't call method on  "+t);return t}},function(t,n,r){var e=r(4);t.exports=function(t,n){return!!t&&e(function(){n?t.call(null,function(){},1):t.call(null)})}},function(t,n,r){var e=r(53),i=r(115),o=r(17),u=r(16),c=r(203);t.exports=function(t,n){var r=1==t,f=2==t,a=3==t,s=4==t,l=6==t,h=5==t||l,v=n||c;return function(n,c,p){for(var d,y,g=o(n),b=i(g),m=e(c,p,3),x=u(b.length),w=0,S=r?v(n,x):f?v(n,0):void 0;x>w;w++)if((h||w in b)&&(d=b[w],y=m(d,w,g),t))if(r)S[w]=y;else if(y)switch(t){case 3:return!0;case 5:return d;case 6:return w;case 2:S.push(d)}else if(s)return!1;return l?-1:a||s?s:S}}},function(t,n,r){var e=r(1),i=r(52),o=r(4);t.exports=function(t,n){var r=(i.Object||{})[t]||Object[t],u={};u[t]=n(r),e(e.S+e.F*o(function(){r(1)}),"Object",u)}},function(t,n,r){var e=r(6);t.exports=function(t,n){if(!e(t))return t;var r,i;if(n&&"function"==typeof(r=t.toString)&&!e(i=r.call(t)))return i;if("function"==typeof(r=t.valueOf)&&!e(i=r.call(t)))return i;if(!n&&"function"==typeof(r=t.toString)&&!e(i=r.call(t)))return i;throw TypeError("Can't convert object to primitive value")}},function(t,n,r){var e=r(5),i=r(25),o=r(91),u=r(13),c="prototype",f=function(t,n,r){var a,s,l,h=t&f.F,v=t&f.G,p=t&f.S,d=t&f.P,y=t&f.B,g=t&f.W,b=v?i:i[n]||(i[n]={}),m=b[c],x=v?e:p?e[n]:(e[n]||{})[c];v&&(r=n);for(a in r)(s=!h&&x&&void 0!==x[a])&&a in b||(l=s?x[a]:r[a],b[a]=v&&"function"!=typeof x[a]?r[a]:y&&s?o(l,e):g&&x[a]==l?function(t){var n=function(n,r,e){if(this instanceof t){switch(arguments.length){case 0:return new t;case 1:return new t(n);case 2:return new t(n,r)}return new t(n,r,e)}return t.apply(this,arguments)};return n[c]=t[c],n}(l):d&&"function"==typeof l?o(Function.call,l):l,d&&((b.virtual||(b.virtual={}))[a]=l,t&f.R&&m&&!m[a]&&u(m,a,l)))};f.F=1,f.G=2,f.S=4,f.P=8,f.B=16,f.W=32,f.U=64,f.R=128,t.exports=f},function(t,n){var r=t.exports={version:"2.4.0"};"number"==typeof __e&&(__e=r)},function(t,n,r){var e=r(26);t.exports=function(t,n,r){if(e(t),void 0===n)return t;switch(r){case 1:return function(r){return t.call(n,r)};case 2:return function(r,e){return t.call(n,r,e)};case 3:return function(r,e,i){return t.call(n,r,e,i)}}return function(){return t.apply(n,arguments)}}},function(t,n,r){var e=r(183),i=r(1),o=r(126)("metadata"),u=o.store||(o.store=new(r(186))),c=function(t,n,r){var i=u.get(t);if(!i){if(!r)return;u.set(t,i=new e)}var o=i.get(n);if(!o){if(!r)return;i.set(n,o=new e)}return o},f=function(t,n,r){var e=c(n,r,!1);return void 0!==e&&e.has(t)},a=function(t,n,r){var e=c(n,r,!1);return void 0===e?void 0:e.get(t)},s=function(t,n,r,e){c(r,e,!0).set(t,n)},l=function(t,n){var r=c(t,n,!1),e=[];return r&&r.forEach(function(t,n){e.push(n)}),e},h=function(t){return void 0===t||"symbol"==typeof t?t:String(t)},v=function(t){i(i.S,"Reflect",t)};t.exports={store:u,map:c,has:f,get:a,set:s,keys:l,key:h,exp:v}},function(t,n,r){"use strict";if(r(10)){var e=r(69),i=r(3),o=r(4),u=r(1),c=r(127),f=r(152),a=r(53),s=r(68),l=r(66),h=r(27),v=r(73),p=r(67),d=r(16),y=r(75),g=r(50),b=r(24),m=r(180),x=r(114),w=r(6),S=r(17),_=r(137),O=r(70),E=r(32),P=r(71).f,j=r(154),F=r(76),M=r(7),A=r(48),N=r(117),T=r(146),I=r(155),k=r(80),L=r(123),R=r(74),C=r(130),D=r(160),U=r(11),W=r(31),G=U.f,B=W.f,V=i.RangeError,z=i.TypeError,q=i.Uint8Array,K="ArrayBuffer",J="Shared"+K,Y="BYTES_PER_ELEMENT",H="prototype",$=Array[H],X=f.ArrayBuffer,Q=f.DataView,Z=A(0),tt=A(2),nt=A(3),rt=A(4),et=A(5),it=A(6),ot=N(!0),ut=N(!1),ct=I.values,ft=I.keys,at=I.entries,st=$.lastIndexOf,lt=$.reduce,ht=$.reduceRight,vt=$.join,pt=$.sort,dt=$.slice,yt=$.toString,gt=$.toLocaleString,bt=M("iterator"),mt=M("toStringTag"),xt=F("typed_constructor"),wt=F("def_constructor"),St=c.CONSTR,_t=c.TYPED,Ot=c.VIEW,Et="Wrong length!",Pt=A(1,function(t,n){return Tt(T(t,t[wt]),n)}),jt=o(function(){return 1===new q(new Uint16Array([1]).buffer)[0]}),Ft=!!q&&!!q[H].set&&o(function(){new q(1).set({})}),Mt=function(t,n){if(void 0===t)throw z(Et);var r=+t,e=d(t);if(n&&!m(r,e))throw V(Et);return e},At=function(t,n){var r=p(t);if(r<0||r%n)throw V("Wrong offset!");return r},Nt=function(t){if(w(t)&&_t in t)return t;throw z(t+" is not a typed array!")},Tt=function(t,n){if(!(w(t)&&xt in t))throw z("It is not a typed array constructor!");return new t(n)},It=function(t,n){return kt(T(t,t[wt]),n)},kt=function(t,n){for(var r=0,e=n.length,i=Tt(t,e);e>r;)i[r]=n[r++];return i},Lt=function(t,n,r){G(t,n,{get:function(){return this._d[r]}})},Rt=function(t){var n,r,e,i,o,u,c=S(t),f=arguments.length,s=f>1?arguments[1]:void 0,l=void 0!==s,h=j(c);if(void 0!=h&&!_(h)){for(u=h.call(c),e=[],n=0;!(o=u.next()).done;n++)e.push(o.value);c=e}for(l&&f>2&&(s=a(s,arguments[2],2)),n=0,r=d(c.length),i=Tt(this,r);r>n;n++)i[n]=l?s(c[n],n):c[n];return i},Ct=function(){for(var t=0,n=arguments.length,r=Tt(this,n);n>t;)r[t]=arguments[t++];return r},Dt=!!q&&o(function(){gt.call(new q(1))}),Ut=function(){return gt.apply(Dt?dt.call(Nt(this)):Nt(this),arguments)},Wt={copyWithin:function(t,n){return D.call(Nt(this),t,n,arguments.length>2?arguments[2]:void 0)},every:function(t){return rt(Nt(this),t,arguments.length>1?arguments[1]:void 0)},fill:function(t){return C.apply(Nt(this),arguments)},filter:function(t){return It(this,tt(Nt(this),t,arguments.length>1?arguments[1]:void 0))},find:function(t){return et(Nt(this),t,arguments.length>1?arguments[1]:void 0)},findIndex:function(t){return it(Nt(this),t,arguments.length>1?arguments[1]:void 0)},forEach:function(t){Z(Nt(this),t,arguments.length>1?arguments[1]:void 0)},indexOf:function(t){return ut(Nt(this),t,arguments.length>1?arguments[1]:void 0)},includes:function(t){return ot(Nt(this),t,arguments.length>1?arguments[1]:void 0)},join:function(t){return vt.apply(Nt(this),arguments)},lastIndexOf:function(t){return st.apply(Nt(this),arguments)},map:function(t){return Pt(Nt(this),t,arguments.length>1?arguments[1]:void 0)},reduce:function(t){return lt.apply(Nt(this),arguments)},reduceRight:function(t){return ht.apply(Nt(this),arguments)},reverse:function(){for(var t,n=this,r=Nt(n).length,e=Math.floor(r/2),i=0;i<e;)t=n[i],n[i++]=n[--r],n[r]=t;return n},some:function(t){return nt(Nt(this),t,arguments.length>1?arguments[1]:void 0)},sort:function(t){return pt.call(Nt(this),t)},subarray:function(t,n){var r=Nt(this),e=r.length,i=y(t,e);return new(T(r,r[wt]))(r.buffer,r.byteOffset+i*r.BYTES_PER_ELEMENT,d((void 0===n?e:y(n,e))-i))}},Gt=function(t,n){return It(this,dt.call(Nt(this),t,n))},Bt=function(t){Nt(this);var n=At(arguments[1],1),r=this.length,e=S(t),i=d(e.length),o=0;if(i+n>r)throw V(Et);for(;o<i;)this[n+o]=e[o++]},Vt={entries:function(){return at.call(Nt(this))},keys:function(){return ft.call(Nt(this))},values:function(){return ct.call(Nt(this))}},zt=function(t,n){return w(t)&&t[_t]&&"symbol"!=typeof n&&n in t&&String(+n)==String(n)},qt=function(t,n){return zt(t,n=g(n,!0))?l(2,t[n]):B(t,n)},Kt=function(t,n,r){return!(zt(t,n=g(n,!0))&&w(r)&&b(r,"value"))||b(r,"get")||b(r,"set")||r.configurable||b(r,"writable")&&!r.writable||b(r,"enumerable")&&!r.enumerable?G(t,n,r):(t[n]=r.value,t)};St||(W.f=qt,U.f=Kt),u(u.S+u.F*!St,"Object",{getOwnPropertyDescriptor:qt,defineProperty:Kt}),o(function(){yt.call({})})&&(yt=gt=function(){return vt.call(this)});var Jt=v({},Wt);v(Jt,Vt),h(Jt,bt,Vt.values),v(Jt,{slice:Gt,set:Bt,constructor:function(){},toString:yt,toLocaleString:Ut}),Lt(Jt,"buffer","b"),Lt(Jt,"byteOffset","o"),Lt(Jt,"byteLength","l"),Lt(Jt,"length","e"),G(Jt,mt,{get:function(){return this[_t]}}),t.exports=function(t,n,r,f){f=!!f;var a=t+(f?"Clamped":"")+"Array",l="Uint8Array"!=a,v="get"+t,p="set"+t,y=i[a],g=y||{},b=y&&E(y),m=!y||!c.ABV,S={},_=y&&y[H],j=function(t,r){var e=t._d;return e.v[v](r*n+e.o,jt)},F=function(t,r,e){var i=t._d;f&&(e=(e=Math.round(e))<0?0:e>255?255:255&e),i.v[p](r*n+i.o,e,jt)},M=function(t,n){G(t,n,{get:function(){return j(this,n)},set:function(t){return F(this,n,t)},enumerable:!0})};m?(y=r(function(t,r,e,i){s(t,y,a,"_d");var o,u,c,f,l=0,v=0;if(w(r)){if(!(r instanceof X||(f=x(r))==K||f==J))return _t in r?kt(y,r):Rt.call(y,r);o=r,v=At(e,n);var p=r.byteLength;if(void 0===i){if(p%n)throw V(Et);if((u=p-v)<0)throw V(Et)}else if((u=d(i)*n)+v>p)throw V(Et);c=u/n}else c=Mt(r,!0),u=c*n,o=new X(u);for(h(t,"_d",{b:o,o:v,l:u,e:c,v:new Q(o)});l<c;)M(t,l++)}),_=y[H]=O(Jt),h(_,"constructor",y)):L(function(t){new y(null),new y(t)},!0)||(y=r(function(t,r,e,i){s(t,y,a);var o;return w(r)?r instanceof X||(o=x(r))==K||o==J?void 0!==i?new g(r,At(e,n),i):void 0!==e?new g(r,At(e,n)):new g(r):_t in r?kt(y,r):Rt.call(y,r):new g(Mt(r,l))}),Z(b!==Function.prototype?P(g).concat(P(b)):P(g),function(t){t in y||h(y,t,g[t])}),y[H]=_,e||(_.constructor=y));var A=_[bt],N=!!A&&("values"==A.name||void 0==A.name),T=Vt.values;h(y,xt,!0),h(_,_t,a),h(_,Ot,!0),h(_,wt,y),(f?new y(1)[mt]==a:mt in _)||G(_,mt,{get:function(){return a}}),S[a]=y,u(u.G+u.W+u.F*(y!=g),S),u(u.S,a,{BYTES_PER_ELEMENT:n,from:Rt,of:Ct}),Y in _||h(_,Y,n),u(u.P,a,Wt),R(a),u(u.P+u.F*Ft,a,{set:Bt}),u(u.P+u.F*!N,a,Vt),u(u.P+u.F*(_.toString!=yt),a,{toString:yt}),u(u.P+u.F*o(function(){new y(1).slice()}),a,{slice:Gt}),u(u.P+u.F*(o(function(){return[1,2].toLocaleString()!=new y([1,2]).toLocaleString()})||!o(function(){_.toLocaleString.call([1,2])})),a,{toLocaleString:Ut}),k[a]=N?A:T,e||N||h(_,bt,T)}}else t.exports=function(){}},function(t,n){var r={}.toString;t.exports=function(t){return r.call(t).slice(8,-1)}},function(t,n,r){var e=r(21),i=r(5).document,o=e(i)&&e(i.createElement);t.exports=function(t){return o?i.createElement(t):{}}},function(t,n,r){t.exports=!r(12)&&!r(18)(function(){return 7!=Object.defineProperty(r(57)("div"),"a",{get:function(){return 7}}).a})},function(t,n,r){"use strict";var e=r(36),i=r(51),o=r(64),u=r(13),c=r(8),f=r(35),a=r(96),s=r(38),l=r(103),h=r(15)("iterator"),v=!([].keys&&"next"in[].keys()),p="keys",d="values",y=function(){return this};t.exports=function(t,n,r,g,b,m,x){a(r,n,g);var w,S,_,O=function(t){if(!v&&t in F)return F[t];switch(t){case p:case d:return function(){return new r(this,t)}}return function(){return new r(this,t)}},E=n+" Iterator",P=b==d,j=!1,F=t.prototype,M=F[h]||F["@@iterator"]||b&&F[b],A=M||O(b),N=b?P?O("entries"):A:void 0,T="Array"==n?F.entries||M:M;if(T&&(_=l(T.call(new t)))!==Object.prototype&&(s(_,E,!0),e||c(_,h)||u(_,h,y)),P&&M&&M.name!==d&&(j=!0,A=function(){return M.call(this)}),e&&!x||!v&&!j&&F[h]||u(F,h,A),f[n]=A,f[E]=y,b)if(w={values:P?A:O(d),keys:m?A:O(p),entries:N},x)for(S in w)S in F||o(F,S,w[S]);else i(i.P+i.F*(v||j),n,w);return w}},function(t,n,r){var e=r(20),i=r(100),o=r(34),u=r(39)("IE_PROTO"),c=function(){},f="prototype",a=function(){var t,n=r(57)("iframe"),e=o.length;for(n.style.display="none",r(93).appendChild(n),n.src="javascript:",t=n.contentWindow.document,t.open(),t.write("<script>document.F=Object<\/script>"),t.close(),a=t.F;e--;)delete a[f][o[e]];return a()};t.exports=Object.create||function(t,n){var r;return null!==t?(c[f]=e(t),r=new c,c[f]=null,r[u]=t):r=a(),void 0===n?r:i(r,n)}},function(t,n,r){var e=r(63),i=r(34).concat("length","prototype");n.f=Object.getOwnPropertyNames||function(t){return e(t,i)}},function(t,n){n.f=Object.getOwnPropertySymbols},function(t,n,r){var e=r(8),i=r(9),o=r(90)(!1),u=r(39)("IE_PROTO");t.exports=function(t,n){var r,c=i(t),f=0,a=[];for(r in c)r!=u&&e(c,r)&&a.push(r);for(;n.length>f;)e(c,r=n[f++])&&(~o(a,r)||a.push(r));return a}},function(t,n,r){t.exports=r(13)},function(t,n,r){var e=r(76)("meta"),i=r(6),o=r(24),u=r(11).f,c=0,f=Object.isExtensible||function(){return!0},a=!r(4)(function(){return f(Object.preventExtensions({}))}),s=function(t){u(t,e,{value:{i:"O"+ ++c,w:{}}})},l=function(t,n){if(!i(t))return"symbol"==typeof t?t:("string"==typeof t?"S":"P")+t;if(!o(t,e)){if(!f(t))return"F";if(!n)return"E";s(t)}return t[e].i},h=function(t,n){if(!o(t,e)){if(!f(t))return!0;if(!n)return!1;s(t)}return t[e].w},v=function(t){return a&&p.NEED&&f(t)&&!o(t,e)&&s(t),t},p=t.exports={KEY:e,NEED:!1,fastKey:l,getWeak:h,onFreeze:v}},function(t,n){t.exports=function(t,n){return{enumerable:!(1&t),configurable:!(2&t),writable:!(4&t),value:n}}},function(t,n){var r=Math.ceil,e=Math.floor;t.exports=function(t){return isNaN(t=+t)?0:(t>0?e:r)(t)}},function(t,n){t.exports=function(t,n,r,e){if(!(t instanceof n)||void 0!==e&&e in t)throw TypeError(r+": incorrect invocation!");return t}},function(t,n){t.exports=!1},function(t,n,r){var e=r(2),i=r(173),o=r(133),u=r(145)("IE_PROTO"),c=function(){},f="prototype",a=function(){var t,n=r(132)("iframe"),e=o.length;for(n.style.display="none",r(135).appendChild(n),n.src="javascript:",t=n.contentWindow.document,t.open(),t.write("<script>document.F=Object<\/script>"),t.close(),a=t.F;e--;)delete a[f][o[e]];return a()};t.exports=Object.create||function(t,n){var r;return null!==t?(c[f]=e(t),r=new c,c[f]=null,r[u]=t):r=a(),void 0===n?r:i(r,n)}},function(t,n,r){var e=r(175),i=r(133).concat("length","prototype");n.f=Object.getOwnPropertyNames||function(t){return e(t,i)}},function(t,n,r){var e=r(175),i=r(133);t.exports=Object.keys||function(t){return e(t,i)}},function(t,n,r){var e=r(28);t.exports=function(t,n,r){for(var i in n)e(t,i,n[i],r);return t}},function(t,n,r){"use strict";var e=r(3),i=r(11),o=r(10),u=r(7)("species");t.exports=function(t){var n=e[t];o&&n&&!n[u]&&i.f(n,u,{configurable:!0,get:function(){return this}})}},function(t,n,r){var e=r(67),i=Math.max,o=Math.min;t.exports=function(t,n){return t=e(t),t<0?i(t+n,0):o(t,n)}},function(t,n){var r=0,e=Math.random();t.exports=function(t){return"Symbol(".concat(void 0===t?"":t,")_",(++r+e).toString(36))}},function(t,n,r){var e=r(33);t.exports=function(t){return Object(e(t))}},function(t,n,r){var e=r(7)("unscopables"),i=Array.prototype;void 0==i[e]&&r(27)(i,e,{}),t.exports=function(t){i[e][t]=!0}},function(t,n,r){var e=r(53),i=r(169),o=r(137),u=r(2),c=r(16),f=r(154),a={},s={},n=t.exports=function(t,n,r,l,h){var v,p,d,y,g=h?function(){return t}:f(t),b=e(r,l,n?2:1),m=0;if("function"!=typeof g)throw TypeError(t+" is not iterable!");if(o(g)){for(v=c(t.length);v>m;m++)if((y=n?b(u(p=t[m])[0],p[1]):b(t[m]))===a||y===s)return y}else for(d=g.call(t);!(p=d.next()).done;)if((y=i(d,b,p.value,n))===a||y===s)return y};n.BREAK=a,n.RETURN=s},function(t,n){t.exports={}},function(t,n,r){var e=r(11).f,i=r(24),o=r(7)("toStringTag");t.exports=function(t,n,r){t&&!i(t=r?t:t.prototype,o)&&e(t,o,{configurable:!0,value:n})}},function(t,n,r){var e=r(1),i=r(46),o=r(4),u=r(150),c="["+u+"]",f="​",a=RegExp("^"+c+c+"*"),s=RegExp(c+c+"*$"),l=function(t,n,r){var i={},c=o(function(){return!!u[t]()||f[t]()!=f}),a=i[t]=c?n(h):u[t];r&&(i[r]=a),e(e.P+e.F*c,"String",i)},h=l.trim=function(t,n){return t=String(i(t)),1&n&&(t=t.replace(a,"")),2&n&&(t=t.replace(s,"")),t};t.exports=l},function(t,n,r){t.exports={default:r(86),__esModule:!0}},function(t,n,r){t.exports={default:r(87),__esModule:!0}},function(t,n,r){"use strict";function e(t){return t&&t.__esModule?t:{default:t}}n.__esModule=!0;var i=r(84),o=e(i),u=r(83),c=e(u),f="function"==typeof c.default&&"symbol"==typeof o.default?function(t){return typeof t}:function(t){return t&&"function"==typeof c.default&&t.constructor===c.default&&t!==c.default.prototype?"symbol":typeof t};n.default="function"==typeof c.default&&"symbol"===f(o.default)?function(t){return void 0===t?"undefined":f(t)}:function(t){return t&&"function"==typeof c.default&&t.constructor===c.default&&t!==c.default.prototype?"symbol":void 0===t?"undefined":f(t)}},function(t,n,r){r(110),r(108),r(111),r(112),t.exports=r(25).Symbol},function(t,n,r){r(109),r(113),t.exports=r(44).f("iterator")},function(t,n){t.exports=function(t){if("function"!=typeof t)throw TypeError(t+" is not a function!");return t}},function(t,n){t.exports=function(){}},function(t,n,r){var e=r(9),i=r(106),o=r(105);t.exports=function(t){return function(n,r,u){var c,f=e(n),a=i(f.length),s=o(u,a);if(t&&r!=r){for(;a>s;)if((c=f[s++])!=c)return!0}else for(;a>s;s++)if((t||s in f)&&f[s]===r)return t||s||0;return!t&&-1}}},function(t,n,r){var e=r(88);t.exports=function(t,n,r){if(e(t),void 0===n)return t;switch(r){case 1:return function(r){return t.call(n,r)};case 2:return function(r,e){return t.call(n,r,e)};case 3:return function(r,e,i){return t.call(n,r,e,i)}}return function(){return t.apply(n,arguments)}}},function(t,n,r){var e=r(19),i=r(62),o=r(37);t.exports=function(t){var n=e(t),r=i.f;if(r)for(var u,c=r(t),f=o.f,a=0;c.length>a;)f.call(t,u=c[a++])&&n.push(u);return n}},function(t,n,r){t.exports=r(5).document&&document.documentElement},function(t,n,r){var e=r(56);t.exports=Object("z").propertyIsEnumerable(0)?Object:function(t){return"String"==e(t)?t.split(""):Object(t)}},function(t,n,r){var e=r(56);t.exports=Array.isArray||function(t){return"Array"==e(t)}},function(t,n,r){"use strict";var e=r(60),i=r(22),o=r(38),u={};r(13)(u,r(15)("iterator"),function(){return this}),t.exports=function(t,n,r){t.prototype=e(u,{next:i(1,r)}),o(t,n+" Iterator")}},function(t,n){t.exports=function(t,n){return{value:n,done:!!t}}},function(t,n,r){var e=r(19),i=r(9);t.exports=function(t,n){for(var r,o=i(t),u=e(o),c=u.length,f=0;c>f;)if(o[r=u[f++]]===n)return r}},function(t,n,r){var e=r(23)("meta"),i=r(21),o=r(8),u=r(14).f,c=0,f=Object.isExtensible||function(){return!0},a=!r(18)(function(){return f(Object.preventExtensions({}))}),s=function(t){u(t,e,{value:{i:"O"+ ++c,w:{}}})},l=function(t,n){if(!i(t))return"symbol"==typeof t?t:("string"==typeof t?"S":"P")+t;if(!o(t,e)){if(!f(t))return"F";if(!n)return"E";s(t)}return t[e].i},h=function(t,n){if(!o(t,e)){if(!f(t))return!0;if(!n)return!1;s(t)}return t[e].w},v=function(t){return a&&p.NEED&&f(t)&&!o(t,e)&&s(t),t},p=t.exports={KEY:e,NEED:!1,fastKey:l,getWeak:h,onFreeze:v}},function(t,n,r){var e=r(14),i=r(20),o=r(19);t.exports=r(12)?Object.defineProperties:function(t,n){i(t);for(var r,u=o(n),c=u.length,f=0;c>f;)e.f(t,r=u[f++],n[r]);return t}},function(t,n,r){var e=r(37),i=r(22),o=r(9),u=r(42),c=r(8),f=r(58),a=Object.getOwnPropertyDescriptor;n.f=r(12)?a:function(t,n){if(t=o(t),n=u(n,!0),f)try{return a(t,n)}catch(t){}if(c(t,n))return i(!e.f.call(t,n),t[n])}},function(t,n,r){var e=r(9),i=r(61).f,o={}.toString,u="object"==typeof window&&window&&Object.getOwnPropertyNames?Object.getOwnPropertyNames(window):[],c=function(t){try{return i(t)}catch(t){return u.slice()}};t.exports.f=function(t){return u&&"[object Window]"==o.call(t)?c(t):i(e(t))}},function(t,n,r){var e=r(8),i=r(77),o=r(39)("IE_PROTO"),u=Object.prototype;t.exports=Object.getPrototypeOf||function(t){return t=i(t),e(t,o)?t[o]:"function"==typeof t.constructor&&t instanceof t.constructor?t.constructor.prototype:t instanceof Object?u:null}},function(t,n,r){var e=r(41),i=r(33);t.exports=function(t){return function(n,r){var o,u,c=String(i(n)),f=e(r),a=c.length;return f<0||f>=a?t?"":void 0:(o=c.charCodeAt(f),o<55296||o>56319||f+1===a||(u=c.charCodeAt(f+1))<56320||u>57343?t?c.charAt(f):o:t?c.slice(f,f+2):u-56320+(o-55296<<10)+65536)}}},function(t,n,r){var e=r(41),i=Math.max,o=Math.min;t.exports=function(t,n){return t=e(t),t<0?i(t+n,0):o(t,n)}},function(t,n,r){var e=r(41),i=Math.min;t.exports=function(t){return t>0?i(e(t),9007199254740991):0}},function(t,n,r){"use strict";var e=r(89),i=r(97),o=r(35),u=r(9);t.exports=r(59)(Array,"Array",function(t,n){this._t=u(t),this._i=0,this._k=n},function(){var t=this._t,n=this._k,r=this._i++;return!t||r>=t.length?(this._t=void 0,i(1)):"keys"==n?i(0,r):"values"==n?i(0,t[r]):i(0,[r,t[r]])},"values"),o.Arguments=o.Array,e("keys"),e("values"),e("entries")},function(t,n){},function(t,n,r){"use strict";var e=r(104)(!0);r(59)(String,"String",function(t){this._t=String(t),this._i=0},function(){var t,n=this._t,r=this._i;return r>=n.length?{value:void 0,done:!0}:(t=e(n,r),this._i+=t.length,{value:t,done:!1})})},function(t,n,r){"use strict";var e=r(5),i=r(8),o=r(12),u=r(51),c=r(64),f=r(99).KEY,a=r(18),s=r(40),l=r(38),h=r(23),v=r(15),p=r(44),d=r(43),y=r(98),g=r(92),b=r(95),m=r(20),x=r(9),w=r(42),S=r(22),_=r(60),O=r(102),E=r(101),P=r(14),j=r(19),F=E.f,M=P.f,A=O.f,N=e.Symbol,T=e.JSON,I=T&&T.stringify,k="prototype",L=v("_hidden"),R=v("toPrimitive"),C={}.propertyIsEnumerable,D=s("symbol-registry"),U=s("symbols"),W=s("op-symbols"),G=Object[k],B="function"==typeof N,V=e.QObject,z=!V||!V[k]||!V[k].findChild,q=o&&a(function(){return 7!=_(M({},"a",{get:function(){return M(this,"a",{value:7}).a}})).a})?function(t,n,r){var e=F(G,n);e&&delete G[n],M(t,n,r),e&&t!==G&&M(G,n,e)}:M,K=function(t){var n=U[t]=_(N[k]);return n._k=t,n},J=B&&"symbol"==typeof N.iterator?function(t){return"symbol"==typeof t}:function(t){return t instanceof N},Y=function(t,n,r){return t===G&&Y(W,n,r),m(t),n=w(n,!0),m(r),i(U,n)?(r.enumerable?(i(t,L)&&t[L][n]&&(t[L][n]=!1),r=_(r,{enumerable:S(0,!1)})):(i(t,L)||M(t,L,S(1,{})),t[L][n]=!0),q(t,n,r)):M(t,n,r)},H=function(t,n){m(t);for(var r,e=g(n=x(n)),i=0,o=e.length;o>i;)Y(t,r=e[i++],n[r]);return t},$=function(t,n){return void 0===n?_(t):H(_(t),n)},X=function(t){var n=C.call(this,t=w(t,!0));return!(this===G&&i(U,t)&&!i(W,t))&&(!(n||!i(this,t)||!i(U,t)||i(this,L)&&this[L][t])||n)},Q=function(t,n){if(t=x(t),n=w(n,!0),t!==G||!i(U,n)||i(W,n)){var r=F(t,n);return!r||!i(U,n)||i(t,L)&&t[L][n]||(r.enumerable=!0),r}},Z=function(t){for(var n,r=A(x(t)),e=[],o=0;r.length>o;)i(U,n=r[o++])||n==L||n==f||e.push(n);return e},tt=function(t){for(var n,r=t===G,e=A(r?W:x(t)),o=[],u=0;e.length>u;)!i(U,n=e[u++])||r&&!i(G,n)||o.push(U[n]);return o};B||(N=function(){if(this instanceof N)throw TypeError("Symbol is not a constructor!");var t=h(arguments.length>0?arguments[0]:void 0),n=function(r){this===G&&n.call(W,r),i(this,L)&&i(this[L],t)&&(this[L][t]=!1),q(this,t,S(1,r))};return o&&z&&q(G,t,{configurable:!0,set:n}),K(t)},c(N[k],"toString",function(){return this._k}),E.f=Q,P.f=Y,r(61).f=O.f=Z,r(37).f=X,r(62).f=tt,o&&!r(36)&&c(G,"propertyIsEnumerable",X,!0),p.f=function(t){return K(v(t))}),u(u.G+u.W+u.F*!B,{Symbol:N});for(var nt="hasInstance,isConcatSpreadable,iterator,match,replace,search,species,split,toPrimitive,toStringTag,unscopables".split(","),rt=0;nt.length>rt;)v(nt[rt++]);for(var nt=j(v.store),rt=0;nt.length>rt;)d(nt[rt++]);u(u.S+u.F*!B,"Symbol",{for:function(t){return i(D,t+="")?D[t]:D[t]=N(t)},keyFor:function(t){if(J(t))return y(D,t);throw TypeError(t+" is not a symbol!")},useSetter:function(){z=!0},useSimple:function(){z=!1}}),u(u.S+u.F*!B,"Object",{create:$,defineProperty:Y,defineProperties:H,getOwnPropertyDescriptor:Q,getOwnPropertyNames:Z,getOwnPropertySymbols:tt}),T&&u(u.S+u.F*(!B||a(function(){var t=N();return"[null]"!=I([t])||"{}"!=I({a:t})||"{}"!=I(Object(t))})),"JSON",{stringify:function(t){if(void 0!==t&&!J(t)){for(var n,r,e=[t],i=1;arguments.length>i;)e.push(arguments[i++]);return n=e[1],"function"==typeof n&&(r=n),!r&&b(n)||(n=function(t,n){if(r&&(n=r.call(this,t,n)),!J(n))return n}),e[1]=n,I.apply(T,e)}}}),N[k][R]||r(13)(N[k],R,N[k].valueOf),l(N,"Symbol"),l(Math,"Math",!0),l(e.JSON,"JSON",!0)},function(t,n,r){r(43)("asyncIterator")},function(t,n,r){r(43)("observable")},function(t,n,r){r(107);for(var e=r(5),i=r(13),o=r(35),u=r(15)("toStringTag"),c=["NodeList","DOMTokenList","MediaList","StyleSheetList","CSSRuleList"],f=0;f<5;f++){var a=c[f],s=e[a],l=s&&s.prototype;l&&!l[u]&&i(l,u,a),o[a]=o.Array}},function(t,n,r){var e=r(45),i=r(7)("toStringTag"),o="Arguments"==e(function(){return arguments}()),u=function(t,n){try{return t[n]}catch(t){}};t.exports=function(t){var n,r,c;return void 0===t?"Undefined":null===t?"Null":"string"==typeof(r=u(n=Object(t),i))?r:o?e(n):"Object"==(c=e(n))&&"function"==typeof n.callee?"Arguments":c}},function(t,n,r){var e=r(45);t.exports=Object("z").propertyIsEnumerable(0)?Object:function(t){return"String"==e(t)?t.split(""):Object(t)}},function(t,n){n.f={}.propertyIsEnumerable},function(t,n,r){var e=r(30),i=r(16),o=r(75);t.exports=function(t){return function(n,r,u){var c,f=e(n),a=i(f.length),s=o(u,a);if(t&&r!=r){for(;a>s;)if((c=f[s++])!=c)return!0}else for(;a>s;s++)if((t||s in f)&&f[s]===r)return t||s||0;return!t&&-1}}},function(t,n,r){"use strict";var e=r(3),i=r(1),o=r(28),u=r(73),c=r(65),f=r(79),a=r(68),s=r(6),l=r(4),h=r(123),v=r(81),p=r(136);t.exports=function(t,n,r,d,y,g){var b=e[t],m=b,x=y?"set":"add",w=m&&m.prototype,S={},_=function(t){var n=w[t];o(w,t,"delete"==t?function(t){return!(g&&!s(t))&&n.call(this,0===t?0:t)}:"has"==t?function(t){return!(g&&!s(t))&&n.call(this,0===t?0:t)}:"get"==t?function(t){return g&&!s(t)?void 0:n.call(this,0===t?0:t)}:"add"==t?function(t){return n.call(this,0===t?0:t),this}:function(t,r){return n.call(this,0===t?0:t,r),this})};if("function"==typeof m&&(g||w.forEach&&!l(function(){(new m).entries().next()}))){var O=new m,E=O[x](g?{}:-0,1)!=O,P=l(function(){O.has(1)}),j=h(function(t){new m(t)}),F=!g&&l(function(){for(var t=new m,n=5;n--;)t[x](n,n);return!t.has(-0)});j||(m=n(function(n,r){a(n,m,t);var e=p(new b,n,m);return void 0!=r&&f(r,y,e[x],e),e}),m.prototype=w,w.constructor=m),(P||F)&&(_("delete"),_("has"),y&&_("get")),(F||E)&&_(x),g&&w.clear&&delete w.clear}else m=d.getConstructor(n,t,y,x),u(m.prototype,r),c.NEED=!0;return v(m,t),S[t]=m,i(i.G+i.W+i.F*(m!=b),S),g||d.setStrong(m,t,y),m}},function(t,n,r){"use strict";var e=r(27),i=r(28),o=r(4),u=r(46),c=r(7);t.exports=function(t,n,r){var f=c(t),a=r(u,f,""[t]),s=a[0],l=a[1];o(function(){var n={};return n[f]=function(){return 7},7!=""[t](n)})&&(i(String.prototype,t,s),e(RegExp.prototype,f,2==n?function(t,n){return l.call(t,this,n)}:function(t){return l.call(t,this)}))}
},function(t,n,r){"use strict";var e=r(2);t.exports=function(){var t=e(this),n="";return t.global&&(n+="g"),t.ignoreCase&&(n+="i"),t.multiline&&(n+="m"),t.unicode&&(n+="u"),t.sticky&&(n+="y"),n}},function(t,n){t.exports=function(t,n,r){var e=void 0===r;switch(n.length){case 0:return e?t():t.call(r);case 1:return e?t(n[0]):t.call(r,n[0]);case 2:return e?t(n[0],n[1]):t.call(r,n[0],n[1]);case 3:return e?t(n[0],n[1],n[2]):t.call(r,n[0],n[1],n[2]);case 4:return e?t(n[0],n[1],n[2],n[3]):t.call(r,n[0],n[1],n[2],n[3])}return t.apply(r,n)}},function(t,n,r){var e=r(6),i=r(45),o=r(7)("match");t.exports=function(t){var n;return e(t)&&(void 0!==(n=t[o])?!!n:"RegExp"==i(t))}},function(t,n,r){var e=r(7)("iterator"),i=!1;try{var o=[7][e]();o.return=function(){i=!0},Array.from(o,function(){throw 2})}catch(t){}t.exports=function(t,n){if(!n&&!i)return!1;var r=!1;try{var o=[7],u=o[e]();u.next=function(){return{done:r=!0}},o[e]=function(){return u},t(o)}catch(t){}return r}},function(t,n,r){t.exports=r(69)||!r(4)(function(){var t=Math.random();__defineSetter__.call(null,t,function(){}),delete r(3)[t]})},function(t,n){n.f=Object.getOwnPropertySymbols},function(t,n,r){var e=r(3),i="__core-js_shared__",o=e[i]||(e[i]={});t.exports=function(t){return o[t]||(o[t]={})}},function(t,n,r){for(var e,i=r(3),o=r(27),u=r(76),c=u("typed_array"),f=u("view"),a=!(!i.ArrayBuffer||!i.DataView),s=a,l=0,h="Int8Array,Uint8Array,Uint8ClampedArray,Int16Array,Uint16Array,Int32Array,Uint32Array,Float32Array,Float64Array".split(",");l<9;)(e=i[h[l++]])?(o(e.prototype,c,!0),o(e.prototype,f,!0)):s=!1;t.exports={ABV:a,CONSTR:s,TYPED:c,VIEW:f}},function(t,n){"use strict";var r={versions:function(){var t=window.navigator.userAgent;return{trident:t.indexOf("Trident")>-1,presto:t.indexOf("Presto")>-1,webKit:t.indexOf("AppleWebKit")>-1,gecko:t.indexOf("Gecko")>-1&&-1==t.indexOf("KHTML"),mobile:!!t.match(/AppleWebKit.*Mobile.*/),ios:!!t.match(/\(i[^;]+;( U;)? CPU.+Mac OS X/),android:t.indexOf("Android")>-1||t.indexOf("Linux")>-1,iPhone:t.indexOf("iPhone")>-1||t.indexOf("Mac")>-1,iPad:t.indexOf("iPad")>-1,webApp:-1==t.indexOf("Safari"),weixin:-1==t.indexOf("MicroMessenger")}}()};t.exports=r},function(t,n,r){"use strict";var e=r(85),i=function(t){return t&&t.__esModule?t:{default:t}}(e),o=function(){function t(t,n,e){return n||e?String.fromCharCode(n||e):r[t]||t}function n(t){return e[t]}var r={"&quot;":'"',"&lt;":"<","&gt;":">","&amp;":"&","&nbsp;":" "},e={};for(var u in r)e[r[u]]=u;return r["&apos;"]="'",e["'"]="&#39;",{encode:function(t){return t?(""+t).replace(/['<> "&]/g,n).replace(/\r?\n/g,"<br/>").replace(/\s/g,"&nbsp;"):""},decode:function(n){return n?(""+n).replace(/<br\s*\/?>/gi,"\n").replace(/&quot;|&lt;|&gt;|&amp;|&nbsp;|&apos;|&#(\d+);|&#(\d+)/g,t).replace(/\u00a0/g," "):""},encodeBase16:function(t){if(!t)return t;t+="";for(var n=[],r=0,e=t.length;e>r;r++)n.push(t.charCodeAt(r).toString(16).toUpperCase());return n.join("")},encodeBase16forJSON:function(t){if(!t)return t;t=t.replace(/[\u4E00-\u9FBF]/gi,function(t){return escape(t).replace("%u","\\u")});for(var n=[],r=0,e=t.length;e>r;r++)n.push(t.charCodeAt(r).toString(16).toUpperCase());return n.join("")},decodeBase16:function(t){if(!t)return t;t+="";for(var n=[],r=0,e=t.length;e>r;r+=2)n.push(String.fromCharCode("0x"+t.slice(r,r+2)));return n.join("")},encodeObject:function(t){if(t instanceof Array)for(var n=0,r=t.length;r>n;n++)t[n]=o.encodeObject(t[n]);else if("object"==(void 0===t?"undefined":(0,i.default)(t)))for(var e in t)t[e]=o.encodeObject(t[e]);else if("string"==typeof t)return o.encode(t);return t},loadScript:function(t){var n=document.createElement("script");document.getElementsByTagName("body")[0].appendChild(n),n.setAttribute("src",t)},addLoadEvent:function(t){var n=window.onload;"function"!=typeof window.onload?window.onload=t:window.onload=function(){n(),t()}}}}();t.exports=o},function(t,n,r){"use strict";var e=r(17),i=r(75),o=r(16);t.exports=function(t){for(var n=e(this),r=o(n.length),u=arguments.length,c=i(u>1?arguments[1]:void 0,r),f=u>2?arguments[2]:void 0,a=void 0===f?r:i(f,r);a>c;)n[c++]=t;return n}},function(t,n,r){"use strict";var e=r(11),i=r(66);t.exports=function(t,n,r){n in t?e.f(t,n,i(0,r)):t[n]=r}},function(t,n,r){var e=r(6),i=r(3).document,o=e(i)&&e(i.createElement);t.exports=function(t){return o?i.createElement(t):{}}},function(t,n){t.exports="constructor,hasOwnProperty,isPrototypeOf,propertyIsEnumerable,toLocaleString,toString,valueOf".split(",")},function(t,n,r){var e=r(7)("match");t.exports=function(t){var n=/./;try{"/./"[t](n)}catch(r){try{return n[e]=!1,!"/./"[t](n)}catch(t){}}return!0}},function(t,n,r){t.exports=r(3).document&&document.documentElement},function(t,n,r){var e=r(6),i=r(144).set;t.exports=function(t,n,r){var o,u=n.constructor;return u!==r&&"function"==typeof u&&(o=u.prototype)!==r.prototype&&e(o)&&i&&i(t,o),t}},function(t,n,r){var e=r(80),i=r(7)("iterator"),o=Array.prototype;t.exports=function(t){return void 0!==t&&(e.Array===t||o[i]===t)}},function(t,n,r){var e=r(45);t.exports=Array.isArray||function(t){return"Array"==e(t)}},function(t,n,r){"use strict";var e=r(70),i=r(66),o=r(81),u={};r(27)(u,r(7)("iterator"),function(){return this}),t.exports=function(t,n,r){t.prototype=e(u,{next:i(1,r)}),o(t,n+" Iterator")}},function(t,n,r){"use strict";var e=r(69),i=r(1),o=r(28),u=r(27),c=r(24),f=r(80),a=r(139),s=r(81),l=r(32),h=r(7)("iterator"),v=!([].keys&&"next"in[].keys()),p="keys",d="values",y=function(){return this};t.exports=function(t,n,r,g,b,m,x){a(r,n,g);var w,S,_,O=function(t){if(!v&&t in F)return F[t];switch(t){case p:case d:return function(){return new r(this,t)}}return function(){return new r(this,t)}},E=n+" Iterator",P=b==d,j=!1,F=t.prototype,M=F[h]||F["@@iterator"]||b&&F[b],A=M||O(b),N=b?P?O("entries"):A:void 0,T="Array"==n?F.entries||M:M;if(T&&(_=l(T.call(new t)))!==Object.prototype&&(s(_,E,!0),e||c(_,h)||u(_,h,y)),P&&M&&M.name!==d&&(j=!0,A=function(){return M.call(this)}),e&&!x||!v&&!j&&F[h]||u(F,h,A),f[n]=A,f[E]=y,b)if(w={values:P?A:O(d),keys:m?A:O(p),entries:N},x)for(S in w)S in F||o(F,S,w[S]);else i(i.P+i.F*(v||j),n,w);return w}},function(t,n){var r=Math.expm1;t.exports=!r||r(10)>22025.465794806718||r(10)<22025.465794806718||-2e-17!=r(-2e-17)?function(t){return 0==(t=+t)?t:t>-1e-6&&t<1e-6?t+t*t/2:Math.exp(t)-1}:r},function(t,n){t.exports=Math.sign||function(t){return 0==(t=+t)||t!=t?t:t<0?-1:1}},function(t,n,r){var e=r(3),i=r(151).set,o=e.MutationObserver||e.WebKitMutationObserver,u=e.process,c=e.Promise,f="process"==r(45)(u);t.exports=function(){var t,n,r,a=function(){var e,i;for(f&&(e=u.domain)&&e.exit();t;){i=t.fn,t=t.next;try{i()}catch(e){throw t?r():n=void 0,e}}n=void 0,e&&e.enter()};if(f)r=function(){u.nextTick(a)};else if(o){var s=!0,l=document.createTextNode("");new o(a).observe(l,{characterData:!0}),r=function(){l.data=s=!s}}else if(c&&c.resolve){var h=c.resolve();r=function(){h.then(a)}}else r=function(){i.call(e,a)};return function(e){var i={fn:e,next:void 0};n&&(n.next=i),t||(t=i,r()),n=i}}},function(t,n,r){var e=r(6),i=r(2),o=function(t,n){if(i(t),!e(n)&&null!==n)throw TypeError(n+": can't set as prototype!")};t.exports={set:Object.setPrototypeOf||("__proto__"in{}?function(t,n,e){try{e=r(53)(Function.call,r(31).f(Object.prototype,"__proto__").set,2),e(t,[]),n=!(t instanceof Array)}catch(t){n=!0}return function(t,r){return o(t,r),n?t.__proto__=r:e(t,r),t}}({},!1):void 0),check:o}},function(t,n,r){var e=r(126)("keys"),i=r(76);t.exports=function(t){return e[t]||(e[t]=i(t))}},function(t,n,r){var e=r(2),i=r(26),o=r(7)("species");t.exports=function(t,n){var r,u=e(t).constructor;return void 0===u||void 0==(r=e(u)[o])?n:i(r)}},function(t,n,r){var e=r(67),i=r(46);t.exports=function(t){return function(n,r){var o,u,c=String(i(n)),f=e(r),a=c.length;return f<0||f>=a?t?"":void 0:(o=c.charCodeAt(f),o<55296||o>56319||f+1===a||(u=c.charCodeAt(f+1))<56320||u>57343?t?c.charAt(f):o:t?c.slice(f,f+2):u-56320+(o-55296<<10)+65536)}}},function(t,n,r){var e=r(122),i=r(46);t.exports=function(t,n,r){if(e(n))throw TypeError("String#"+r+" doesn't accept regex!");return String(i(t))}},function(t,n,r){"use strict";var e=r(67),i=r(46);t.exports=function(t){var n=String(i(this)),r="",o=e(t);if(o<0||o==1/0)throw RangeError("Count can't be negative");for(;o>0;(o>>>=1)&&(n+=n))1&o&&(r+=n);return r}},function(t,n){t.exports="\t\n\v\f\r   ᠎             　\u2028\u2029\ufeff"},function(t,n,r){var e,i,o,u=r(53),c=r(121),f=r(135),a=r(132),s=r(3),l=s.process,h=s.setImmediate,v=s.clearImmediate,p=s.MessageChannel,d=0,y={},g="onreadystatechange",b=function(){var t=+this;if(y.hasOwnProperty(t)){var n=y[t];delete y[t],n()}},m=function(t){b.call(t.data)};h&&v||(h=function(t){for(var n=[],r=1;arguments.length>r;)n.push(arguments[r++]);return y[++d]=function(){c("function"==typeof t?t:Function(t),n)},e(d),d},v=function(t){delete y[t]},"process"==r(45)(l)?e=function(t){l.nextTick(u(b,t,1))}:p?(i=new p,o=i.port2,i.port1.onmessage=m,e=u(o.postMessage,o,1)):s.addEventListener&&"function"==typeof postMessage&&!s.importScripts?(e=function(t){s.postMessage(t+"","*")},s.addEventListener("message",m,!1)):e=g in a("script")?function(t){f.appendChild(a("script"))[g]=function(){f.removeChild(this),b.call(t)}}:function(t){setTimeout(u(b,t,1),0)}),t.exports={set:h,clear:v}},function(t,n,r){"use strict";var e=r(3),i=r(10),o=r(69),u=r(127),c=r(27),f=r(73),a=r(4),s=r(68),l=r(67),h=r(16),v=r(71).f,p=r(11).f,d=r(130),y=r(81),g="ArrayBuffer",b="DataView",m="prototype",x="Wrong length!",w="Wrong index!",S=e[g],_=e[b],O=e.Math,E=e.RangeError,P=e.Infinity,j=S,F=O.abs,M=O.pow,A=O.floor,N=O.log,T=O.LN2,I="buffer",k="byteLength",L="byteOffset",R=i?"_b":I,C=i?"_l":k,D=i?"_o":L,U=function(t,n,r){var e,i,o,u=Array(r),c=8*r-n-1,f=(1<<c)-1,a=f>>1,s=23===n?M(2,-24)-M(2,-77):0,l=0,h=t<0||0===t&&1/t<0?1:0;for(t=F(t),t!=t||t===P?(i=t!=t?1:0,e=f):(e=A(N(t)/T),t*(o=M(2,-e))<1&&(e--,o*=2),t+=e+a>=1?s/o:s*M(2,1-a),t*o>=2&&(e++,o/=2),e+a>=f?(i=0,e=f):e+a>=1?(i=(t*o-1)*M(2,n),e+=a):(i=t*M(2,a-1)*M(2,n),e=0));n>=8;u[l++]=255&i,i/=256,n-=8);for(e=e<<n|i,c+=n;c>0;u[l++]=255&e,e/=256,c-=8);return u[--l]|=128*h,u},W=function(t,n,r){var e,i=8*r-n-1,o=(1<<i)-1,u=o>>1,c=i-7,f=r-1,a=t[f--],s=127&a;for(a>>=7;c>0;s=256*s+t[f],f--,c-=8);for(e=s&(1<<-c)-1,s>>=-c,c+=n;c>0;e=256*e+t[f],f--,c-=8);if(0===s)s=1-u;else{if(s===o)return e?NaN:a?-P:P;e+=M(2,n),s-=u}return(a?-1:1)*e*M(2,s-n)},G=function(t){return t[3]<<24|t[2]<<16|t[1]<<8|t[0]},B=function(t){return[255&t]},V=function(t){return[255&t,t>>8&255]},z=function(t){return[255&t,t>>8&255,t>>16&255,t>>24&255]},q=function(t){return U(t,52,8)},K=function(t){return U(t,23,4)},J=function(t,n,r){p(t[m],n,{get:function(){return this[r]}})},Y=function(t,n,r,e){var i=+r,o=l(i);if(i!=o||o<0||o+n>t[C])throw E(w);var u=t[R]._b,c=o+t[D],f=u.slice(c,c+n);return e?f:f.reverse()},H=function(t,n,r,e,i,o){var u=+r,c=l(u);if(u!=c||c<0||c+n>t[C])throw E(w);for(var f=t[R]._b,a=c+t[D],s=e(+i),h=0;h<n;h++)f[a+h]=s[o?h:n-h-1]},$=function(t,n){s(t,S,g);var r=+n,e=h(r);if(r!=e)throw E(x);return e};if(u.ABV){if(!a(function(){new S})||!a(function(){new S(.5)})){S=function(t){return new j($(this,t))};for(var X,Q=S[m]=j[m],Z=v(j),tt=0;Z.length>tt;)(X=Z[tt++])in S||c(S,X,j[X]);o||(Q.constructor=S)}var nt=new _(new S(2)),rt=_[m].setInt8;nt.setInt8(0,2147483648),nt.setInt8(1,2147483649),!nt.getInt8(0)&&nt.getInt8(1)||f(_[m],{setInt8:function(t,n){rt.call(this,t,n<<24>>24)},setUint8:function(t,n){rt.call(this,t,n<<24>>24)}},!0)}else S=function(t){var n=$(this,t);this._b=d.call(Array(n),0),this[C]=n},_=function(t,n,r){s(this,_,b),s(t,S,b);var e=t[C],i=l(n);if(i<0||i>e)throw E("Wrong offset!");if(r=void 0===r?e-i:h(r),i+r>e)throw E(x);this[R]=t,this[D]=i,this[C]=r},i&&(J(S,k,"_l"),J(_,I,"_b"),J(_,k,"_l"),J(_,L,"_o")),f(_[m],{getInt8:function(t){return Y(this,1,t)[0]<<24>>24},getUint8:function(t){return Y(this,1,t)[0]},getInt16:function(t){var n=Y(this,2,t,arguments[1]);return(n[1]<<8|n[0])<<16>>16},getUint16:function(t){var n=Y(this,2,t,arguments[1]);return n[1]<<8|n[0]},getInt32:function(t){return G(Y(this,4,t,arguments[1]))},getUint32:function(t){return G(Y(this,4,t,arguments[1]))>>>0},getFloat32:function(t){return W(Y(this,4,t,arguments[1]),23,4)},getFloat64:function(t){return W(Y(this,8,t,arguments[1]),52,8)},setInt8:function(t,n){H(this,1,t,B,n)},setUint8:function(t,n){H(this,1,t,B,n)},setInt16:function(t,n){H(this,2,t,V,n,arguments[2])},setUint16:function(t,n){H(this,2,t,V,n,arguments[2])},setInt32:function(t,n){H(this,4,t,z,n,arguments[2])},setUint32:function(t,n){H(this,4,t,z,n,arguments[2])},setFloat32:function(t,n){H(this,4,t,K,n,arguments[2])},setFloat64:function(t,n){H(this,8,t,q,n,arguments[2])}});y(S,g),y(_,b),c(_[m],u.VIEW,!0),n[g]=S,n[b]=_},function(t,n,r){var e=r(3),i=r(52),o=r(69),u=r(182),c=r(11).f;t.exports=function(t){var n=i.Symbol||(i.Symbol=o?{}:e.Symbol||{});"_"==t.charAt(0)||t in n||c(n,t,{value:u.f(t)})}},function(t,n,r){var e=r(114),i=r(7)("iterator"),o=r(80);t.exports=r(52).getIteratorMethod=function(t){if(void 0!=t)return t[i]||t["@@iterator"]||o[e(t)]}},function(t,n,r){"use strict";var e=r(78),i=r(170),o=r(80),u=r(30);t.exports=r(140)(Array,"Array",function(t,n){this._t=u(t),this._i=0,this._k=n},function(){var t=this._t,n=this._k,r=this._i++;return!t||r>=t.length?(this._t=void 0,i(1)):"keys"==n?i(0,r):"values"==n?i(0,t[r]):i(0,[r,t[r]])},"values"),o.Arguments=o.Array,e("keys"),e("values"),e("entries")},function(t,n){function r(t,n){t.classList?t.classList.add(n):t.className+=" "+n}t.exports=r},function(t,n){function r(t,n){if(t.classList)t.classList.remove(n);else{var r=new RegExp("(^|\\b)"+n.split(" ").join("|")+"(\\b|$)","gi");t.className=t.className.replace(r," ")}}t.exports=r},function(t,n){function r(){throw new Error("setTimeout has not been defined")}function e(){throw new Error("clearTimeout has not been defined")}function i(t){if(s===setTimeout)return setTimeout(t,0);if((s===r||!s)&&setTimeout)return s=setTimeout,setTimeout(t,0);try{return s(t,0)}catch(n){try{return s.call(null,t,0)}catch(n){return s.call(this,t,0)}}}function o(t){if(l===clearTimeout)return clearTimeout(t);if((l===e||!l)&&clearTimeout)return l=clearTimeout,clearTimeout(t);try{return l(t)}catch(n){try{return l.call(null,t)}catch(n){return l.call(this,t)}}}function u(){d&&v&&(d=!1,v.length?p=v.concat(p):y=-1,p.length&&c())}function c(){if(!d){var t=i(u);d=!0;for(var n=p.length;n;){for(v=p,p=[];++y<n;)v&&v[y].run();y=-1,n=p.length}v=null,d=!1,o(t)}}function f(t,n){this.fun=t,this.array=n}function a(){}var s,l,h=t.exports={};!function(){try{s="function"==typeof setTimeout?setTimeout:r}catch(t){s=r}try{l="function"==typeof clearTimeout?clearTimeout:e}catch(t){l=e}}();var v,p=[],d=!1,y=-1;h.nextTick=function(t){var n=new Array(arguments.length-1);if(arguments.length>1)for(var r=1;r<arguments.length;r++)n[r-1]=arguments[r];p.push(new f(t,n)),1!==p.length||d||i(c)},f.prototype.run=function(){this.fun.apply(null,this.array)},h.title="browser",h.browser=!0,h.env={},h.argv=[],h.version="",h.versions={},h.on=a,h.addListener=a,h.once=a,h.off=a,h.removeListener=a,h.removeAllListeners=a,h.emit=a,h.prependListener=a,h.prependOnceListener=a,h.listeners=function(t){return[]},h.binding=function(t){throw new Error("process.binding is not supported")},h.cwd=function(){return"/"},h.chdir=function(t){throw new Error("process.chdir is not supported")},h.umask=function(){return 0}},function(t,n,r){var e=r(45);t.exports=function(t,n){if("number"!=typeof t&&"Number"!=e(t))throw TypeError(n);return+t}},function(t,n,r){"use strict";var e=r(17),i=r(75),o=r(16);t.exports=[].copyWithin||function(t,n){var r=e(this),u=o(r.length),c=i(t,u),f=i(n,u),a=arguments.length>2?arguments[2]:void 0,s=Math.min((void 0===a?u:i(a,u))-f,u-c),l=1;for(f<c&&c<f+s&&(l=-1,f+=s-1,c+=s-1);s-- >0;)f in r?r[c]=r[f]:delete r[c],c+=l,f+=l;return r}},function(t,n,r){var e=r(79);t.exports=function(t,n){var r=[];return e(t,!1,r.push,r,n),r}},function(t,n,r){var e=r(26),i=r(17),o=r(115),u=r(16);t.exports=function(t,n,r,c,f){e(n);var a=i(t),s=o(a),l=u(a.length),h=f?l-1:0,v=f?-1:1;if(r<2)for(;;){if(h in s){c=s[h],h+=v;break}if(h+=v,f?h<0:l<=h)throw TypeError("Reduce of empty array with no initial value")}for(;f?h>=0:l>h;h+=v)h in s&&(c=n(c,s[h],h,a));return c}},function(t,n,r){"use strict";var e=r(26),i=r(6),o=r(121),u=[].slice,c={},f=function(t,n,r){if(!(n in c)){for(var e=[],i=0;i<n;i++)e[i]="a["+i+"]";c[n]=Function("F,a","return new F("+e.join(",")+")")}return c[n](t,r)};t.exports=Function.bind||function(t){var n=e(this),r=u.call(arguments,1),c=function(){var e=r.concat(u.call(arguments));return this instanceof c?f(n,e.length,e):o(n,e,t)};return i(n.prototype)&&(c.prototype=n.prototype),c}},function(t,n,r){"use strict";var e=r(11).f,i=r(70),o=r(73),u=r(53),c=r(68),f=r(46),a=r(79),s=r(140),l=r(170),h=r(74),v=r(10),p=r(65).fastKey,d=v?"_s":"size",y=function(t,n){var r,e=p(n);if("F"!==e)return t._i[e];for(r=t._f;r;r=r.n)if(r.k==n)return r};t.exports={getConstructor:function(t,n,r,s){var l=t(function(t,e){c(t,l,n,"_i"),t._i=i(null),t._f=void 0,t._l=void 0,t[d]=0,void 0!=e&&a(e,r,t[s],t)});return o(l.prototype,{clear:function(){for(var t=this,n=t._i,r=t._f;r;r=r.n)r.r=!0,r.p&&(r.p=r.p.n=void 0),delete n[r.i];t._f=t._l=void 0,t[d]=0},delete:function(t){var n=this,r=y(n,t);if(r){var e=r.n,i=r.p;delete n._i[r.i],r.r=!0,i&&(i.n=e),e&&(e.p=i),n._f==r&&(n._f=e),n._l==r&&(n._l=i),n[d]--}return!!r},forEach:function(t){c(this,l,"forEach");for(var n,r=u(t,arguments.length>1?arguments[1]:void 0,3);n=n?n.n:this._f;)for(r(n.v,n.k,this);n&&n.r;)n=n.p},has:function(t){return!!y(this,t)}}),v&&e(l.prototype,"size",{get:function(){return f(this[d])}}),l},def:function(t,n,r){var e,i,o=y(t,n);return o?o.v=r:(t._l=o={i:i=p(n,!0),k:n,v:r,p:e=t._l,n:void 0,r:!1},t._f||(t._f=o),e&&(e.n=o),t[d]++,"F"!==i&&(t._i[i]=o)),t},getEntry:y,setStrong:function(t,n,r){s(t,n,function(t,n){this._t=t,this._k=n,this._l=void 0},function(){for(var t=this,n=t._k,r=t._l;r&&r.r;)r=r.p;return t._t&&(t._l=r=r?r.n:t._t._f)?"keys"==n?l(0,r.k):"values"==n?l(0,r.v):l(0,[r.k,r.v]):(t._t=void 0,l(1))},r?"entries":"values",!r,!0),h(n)}}},function(t,n,r){var e=r(114),i=r(161);t.exports=function(t){return function(){if(e(this)!=t)throw TypeError(t+"#toJSON isn't generic");return i(this)}}},function(t,n,r){"use strict";var e=r(73),i=r(65).getWeak,o=r(2),u=r(6),c=r(68),f=r(79),a=r(48),s=r(24),l=a(5),h=a(6),v=0,p=function(t){return t._l||(t._l=new d)},d=function(){this.a=[]},y=function(t,n){return l(t.a,function(t){return t[0]===n})};d.prototype={get:function(t){var n=y(this,t);if(n)return n[1]},has:function(t){return!!y(this,t)},set:function(t,n){var r=y(this,t);r?r[1]=n:this.a.push([t,n])},delete:function(t){var n=h(this.a,function(n){return n[0]===t});return~n&&this.a.splice(n,1),!!~n}},t.exports={getConstructor:function(t,n,r,o){var a=t(function(t,e){c(t,a,n,"_i"),t._i=v++,t._l=void 0,void 0!=e&&f(e,r,t[o],t)});return e(a.prototype,{delete:function(t){if(!u(t))return!1;var n=i(t);return!0===n?p(this).delete(t):n&&s(n,this._i)&&delete n[this._i]},has:function(t){if(!u(t))return!1;var n=i(t);return!0===n?p(this).has(t):n&&s(n,this._i)}}),a},def:function(t,n,r){var e=i(o(n),!0);return!0===e?p(t).set(n,r):e[t._i]=r,t},ufstore:p}},function(t,n,r){t.exports=!r(10)&&!r(4)(function(){return 7!=Object.defineProperty(r(132)("div"),"a",{get:function(){return 7}}).a})},function(t,n,r){var e=r(6),i=Math.floor;t.exports=function(t){return!e(t)&&isFinite(t)&&i(t)===t}},function(t,n,r){var e=r(2);t.exports=function(t,n,r,i){try{return i?n(e(r)[0],r[1]):n(r)}catch(n){var o=t.return;throw void 0!==o&&e(o.call(t)),n}}},function(t,n){t.exports=function(t,n){return{value:n,done:!!t}}},function(t,n){t.exports=Math.log1p||function(t){return(t=+t)>-1e-8&&t<1e-8?t-t*t/2:Math.log(1+t)}},function(t,n,r){"use strict";var e=r(72),i=r(125),o=r(116),u=r(17),c=r(115),f=Object.assign;t.exports=!f||r(4)(function(){var t={},n={},r=Symbol(),e="abcdefghijklmnopqrst";return t[r]=7,e.split("").forEach(function(t){n[t]=t}),7!=f({},t)[r]||Object.keys(f({},n)).join("")!=e})?function(t,n){for(var r=u(t),f=arguments.length,a=1,s=i.f,l=o.f;f>a;)for(var h,v=c(arguments[a++]),p=s?e(v).concat(s(v)):e(v),d=p.length,y=0;d>y;)l.call(v,h=p[y++])&&(r[h]=v[h]);return r}:f},function(t,n,r){var e=r(11),i=r(2),o=r(72);t.exports=r(10)?Object.defineProperties:function(t,n){i(t);for(var r,u=o(n),c=u.length,f=0;c>f;)e.f(t,r=u[f++],n[r]);return t}},function(t,n,r){var e=r(30),i=r(71).f,o={}.toString,u="object"==typeof window&&window&&Object.getOwnPropertyNames?Object.getOwnPropertyNames(window):[],c=function(t){try{return i(t)}catch(t){return u.slice()}};t.exports.f=function(t){return u&&"[object Window]"==o.call(t)?c(t):i(e(t))}},function(t,n,r){var e=r(24),i=r(30),o=r(117)(!1),u=r(145)("IE_PROTO");t.exports=function(t,n){var r,c=i(t),f=0,a=[];for(r in c)r!=u&&e(c,r)&&a.push(r);for(;n.length>f;)e(c,r=n[f++])&&(~o(a,r)||a.push(r));return a}},function(t,n,r){var e=r(72),i=r(30),o=r(116).f;t.exports=function(t){return function(n){for(var r,u=i(n),c=e(u),f=c.length,a=0,s=[];f>a;)o.call(u,r=c[a++])&&s.push(t?[r,u[r]]:u[r]);return s}}},function(t,n,r){var e=r(71),i=r(125),o=r(2),u=r(3).Reflect;t.exports=u&&u.ownKeys||function(t){var n=e.f(o(t)),r=i.f;return r?n.concat(r(t)):n}},function(t,n,r){var e=r(3).parseFloat,i=r(82).trim;t.exports=1/e(r(150)+"-0")!=-1/0?function(t){var n=i(String(t),3),r=e(n);return 0===r&&"-"==n.charAt(0)?-0:r}:e},function(t,n,r){var e=r(3).parseInt,i=r(82).trim,o=r(150),u=/^[\-+]?0[xX]/;t.exports=8!==e(o+"08")||22!==e(o+"0x16")?function(t,n){var r=i(String(t),3);return e(r,n>>>0||(u.test(r)?16:10))}:e},function(t,n){t.exports=Object.is||function(t,n){return t===n?0!==t||1/t==1/n:t!=t&&n!=n}},function(t,n,r){var e=r(16),i=r(149),o=r(46);t.exports=function(t,n,r,u){var c=String(o(t)),f=c.length,a=void 0===r?" ":String(r),s=e(n);if(s<=f||""==a)return c;var l=s-f,h=i.call(a,Math.ceil(l/a.length));return h.length>l&&(h=h.slice(0,l)),u?h+c:c+h}},function(t,n,r){n.f=r(7)},function(t,n,r){"use strict";var e=r(164);t.exports=r(118)("Map",function(t){return function(){return t(this,arguments.length>0?arguments[0]:void 0)}},{get:function(t){var n=e.getEntry(this,t);return n&&n.v},set:function(t,n){return e.def(this,0===t?0:t,n)}},e,!0)},function(t,n,r){r(10)&&"g"!=/./g.flags&&r(11).f(RegExp.prototype,"flags",{configurable:!0,get:r(120)})},function(t,n,r){"use strict";var e=r(164);t.exports=r(118)("Set",function(t){return function(){return t(this,arguments.length>0?arguments[0]:void 0)}},{add:function(t){return e.def(this,t=0===t?0:t,t)}},e)},function(t,n,r){"use strict";var e,i=r(48)(0),o=r(28),u=r(65),c=r(172),f=r(166),a=r(6),s=u.getWeak,l=Object.isExtensible,h=f.ufstore,v={},p=function(t){return function(){return t(this,arguments.length>0?arguments[0]:void 0)}},d={get:function(t){if(a(t)){var n=s(t);return!0===n?h(this).get(t):n?n[this._i]:void 0}},set:function(t,n){return f.def(this,t,n)}},y=t.exports=r(118)("WeakMap",p,d,f,!0,!0);7!=(new y).set((Object.freeze||Object)(v),7).get(v)&&(e=f.getConstructor(p),c(e.prototype,d),u.NEED=!0,i(["delete","has","get","set"],function(t){var n=y.prototype,r=n[t];o(n,t,function(n,i){if(a(n)&&!l(n)){this._f||(this._f=new e);var o=this._f[t](n,i);return"set"==t?this:o}return r.call(this,n,i)})}))},,,,function(t,n){"use strict";function r(){var t=document.querySelector("#page-nav");if(t&&!document.querySelector("#page-nav .extend.prev")&&(t.innerHTML='<a class="extend prev disabled" rel="prev">&laquo; Prev</a>'+t.innerHTML),t&&!document.querySelector("#page-nav .extend.next")&&(t.innerHTML=t.innerHTML+'<a class="extend next disabled" rel="next">Next &raquo;</a>'),yiliaConfig&&yiliaConfig.open_in_new){document.querySelectorAll(".article-entry a:not(.article-more-a)").forEach(function(t){var n=t.getAttribute("target");n&&""!==n||t.setAttribute("target","_blank")})}if(yiliaConfig&&yiliaConfig.toc_hide_index){document.querySelectorAll(".toc-number").forEach(function(t){t.style.display="none"})}var n=document.querySelector("#js-aboutme");n&&0!==n.length&&(n.innerHTML=n.innerText)}t.exports={init:r}},function(t,n,r){"use strict";function e(t){return t&&t.__esModule?t:{default:t}}function i(t,n){var r=/\/|index.html/g;return t.replace(r,"")===n.replace(r,"")}function o(){for(var t=document.querySelectorAll(".js-header-menu li a"),n=window.location.pathname,r=0,e=t.length;r<e;r++){var o=t[r];i(n,o.getAttribute("href"))&&(0,h.default)(o,"active")}}function u(t){for(var n=t.offsetLeft,r=t.offsetParent;null!==r;)n+=r.offsetLeft,r=r.offsetParent;return n}function c(t){for(var n=t.offsetTop,r=t.offsetParent;null!==r;)n+=r.offsetTop,r=r.offsetParent;return n}function f(t,n,r,e,i){var o=u(t),f=c(t)-n;if(f-r<=i){var a=t.$newDom;a||(a=t.cloneNode(!0),(0,d.default)(t,a),t.$newDom=a,a.style.position="fixed",a.style.top=(r||f)+"px",a.style.left=o+"px",a.style.zIndex=e||2,a.style.width="100%",a.style.color="#fff"),a.style.visibility="visible",t.style.visibility="hidden"}else{t.style.visibility="visible";var s=t.$newDom;s&&(s.style.visibility="hidden")}}function a(){var t=document.querySelector(".js-overlay"),n=document.querySelector(".js-header-menu");f(t,document.body.scrollTop,-63,2,0),f(n,document.body.scrollTop,1,3,0)}function s(){document.querySelector("#container").addEventListener("scroll",function(t){a()}),window.addEventListener("scroll",function(t){a()}),a()}var l=r(156),h=e(l),v=r(157),p=(e(v),r(382)),d=e(p),y=r(128),g=e(y),b=r(190),m=e(b),x=r(129);(function(){g.default.versions.mobile&&window.screen.width<800&&(o(),s())})(),(0,x.addLoadEvent)(function(){m.default.init()}),t.exports={}},,,,function(t,n,r){(function(t){"use strict";function n(t,n,r){t[n]||Object[e](t,n,{writable:!0,configurable:!0,value:r})}if(r(381),r(391),r(198),t._babelPolyfill)throw new Error("only one instance of babel-polyfill is allowed");t._babelPolyfill=!0;var e="defineProperty";n(String.prototype,"padLeft","".padStart),n(String.prototype,"padRight","".padEnd),"pop,reverse,shift,keys,values,entries,indexOf,every,some,forEach,map,filter,find,findIndex,includes,join,slice,concat,push,splice,unshift,sort,lastIndexOf,reduce,reduceRight,copyWithin,fill".split(",").forEach(function(t){[][t]&&n(Array,t,Function.call.bind([][t]))})}).call(n,function(){return this}())},,,function(t,n,r){r(210),t.exports=r(52).RegExp.escape},,,,function(t,n,r){var e=r(6),i=r(138),o=r(7)("species");t.exports=function(t){var n;return i(t)&&(n=t.constructor,"function"!=typeof n||n!==Array&&!i(n.prototype)||(n=void 0),e(n)&&null===(n=n[o])&&(n=void 0)),void 0===n?Array:n}},function(t,n,r){var e=r(202);t.exports=function(t,n){return new(e(t))(n)}},function(t,n,r){"use strict";var e=r(2),i=r(50),o="number";t.exports=function(t){if("string"!==t&&t!==o&&"default"!==t)throw TypeError("Incorrect hint");return i(e(this),t!=o)}},function(t,n,r){var e=r(72),i=r(125),o=r(116);t.exports=function(t){var n=e(t),r=i.f;if(r)for(var u,c=r(t),f=o.f,a=0;c.length>a;)f.call(t,u=c[a++])&&n.push(u);return n}},function(t,n,r){var e=r(72),i=r(30);t.exports=function(t,n){for(var r,o=i(t),u=e(o),c=u.length,f=0;c>f;)if(o[r=u[f++]]===n)return r}},function(t,n,r){"use strict";var e=r(208),i=r(121),o=r(26);t.exports=function(){for(var t=o(this),n=arguments.length,r=Array(n),u=0,c=e._,f=!1;n>u;)(r[u]=arguments[u++])===c&&(f=!0);return function(){var e,o=this,u=arguments.length,a=0,s=0;if(!f&&!u)return i(t,r,o);if(e=r.slice(),f)for(;n>a;a++)e[a]===c&&(e[a]=arguments[s++]);for(;u>s;)e.push(arguments[s++]);return i(t,e,o)}}},function(t,n,r){t.exports=r(3)},function(t,n){t.exports=function(t,n){var r=n===Object(n)?function(t){return n[t]}:n;return function(n){return String(n).replace(t,r)}}},function(t,n,r){var e=r(1),i=r(209)(/[\\^$*+?.()|[\]{}]/g,"\\$&");e(e.S,"RegExp",{escape:function(t){return i(t)}})},function(t,n,r){var e=r(1);e(e.P,"Array",{copyWithin:r(160)}),r(78)("copyWithin")},function(t,n,r){"use strict";var e=r(1),i=r(48)(4);e(e.P+e.F*!r(47)([].every,!0),"Array",{every:function(t){return i(this,t,arguments[1])}})},function(t,n,r){var e=r(1);e(e.P,"Array",{fill:r(130)}),r(78)("fill")},function(t,n,r){"use strict";var e=r(1),i=r(48)(2);e(e.P+e.F*!r(47)([].filter,!0),"Array",{filter:function(t){return i(this,t,arguments[1])}})},function(t,n,r){"use strict";var e=r(1),i=r(48)(6),o="findIndex",u=!0;o in[]&&Array(1)[o](function(){u=!1}),e(e.P+e.F*u,"Array",{findIndex:function(t){return i(this,t,arguments.length>1?arguments[1]:void 0)}}),r(78)(o)},function(t,n,r){"use strict";var e=r(1),i=r(48)(5),o="find",u=!0;o in[]&&Array(1)[o](function(){u=!1}),e(e.P+e.F*u,"Array",{find:function(t){return i(this,t,arguments.length>1?arguments[1]:void 0)}}),r(78)(o)},function(t,n,r){"use strict";var e=r(1),i=r(48)(0),o=r(47)([].forEach,!0);e(e.P+e.F*!o,"Array",{forEach:function(t){return i(this,t,arguments[1])}})},function(t,n,r){"use strict";var e=r(53),i=r(1),o=r(17),u=r(169),c=r(137),f=r(16),a=r(131),s=r(154);i(i.S+i.F*!r(123)(function(t){Array.from(t)}),"Array",{from:function(t){var n,r,i,l,h=o(t),v="function"==typeof this?this:Array,p=arguments.length,d=p>1?arguments[1]:void 0,y=void 0!==d,g=0,b=s(h);if(y&&(d=e(d,p>2?arguments[2]:void 0,2)),void 0==b||v==Array&&c(b))for(n=f(h.length),r=new v(n);n>g;g++)a(r,g,y?d(h[g],g):h[g]);else for(l=b.call(h),r=new v;!(i=l.next()).done;g++)a(r,g,y?u(l,d,[i.value,g],!0):i.value);return r.length=g,r}})},function(t,n,r){"use strict";var e=r(1),i=r(117)(!1),o=[].indexOf,u=!!o&&1/[1].indexOf(1,-0)<0;e(e.P+e.F*(u||!r(47)(o)),"Array",{indexOf:function(t){return u?o.apply(this,arguments)||0:i(this,t,arguments[1])}})},function(t,n,r){var e=r(1);e(e.S,"Array",{isArray:r(138)})},function(t,n,r){"use strict";var e=r(1),i=r(30),o=[].join;e(e.P+e.F*(r(115)!=Object||!r(47)(o)),"Array",{join:function(t){return o.call(i(this),void 0===t?",":t)}})},function(t,n,r){"use strict";var e=r(1),i=r(30),o=r(67),u=r(16),c=[].lastIndexOf,f=!!c&&1/[1].lastIndexOf(1,-0)<0;e(e.P+e.F*(f||!r(47)(c)),"Array",{lastIndexOf:function(t){if(f)return c.apply(this,arguments)||0;var n=i(this),r=u(n.length),e=r-1;for(arguments.length>1&&(e=Math.min(e,o(arguments[1]))),e<0&&(e=r+e);e>=0;e--)if(e in n&&n[e]===t)return e||0;return-1}})},function(t,n,r){"use strict";var e=r(1),i=r(48)(1);e(e.P+e.F*!r(47)([].map,!0),"Array",{map:function(t){return i(this,t,arguments[1])}})},function(t,n,r){"use strict";var e=r(1),i=r(131);e(e.S+e.F*r(4)(function(){function t(){}return!(Array.of.call(t)instanceof t)}),"Array",{of:function(){for(var t=0,n=arguments.length,r=new("function"==typeof this?this:Array)(n);n>t;)i(r,t,arguments[t++]);return r.length=n,r}})},function(t,n,r){"use strict";var e=r(1),i=r(162);e(e.P+e.F*!r(47)([].reduceRight,!0),"Array",{reduceRight:function(t){return i(this,t,arguments.length,arguments[1],!0)}})},function(t,n,r){"use strict";var e=r(1),i=r(162);e(e.P+e.F*!r(47)([].reduce,!0),"Array",{reduce:function(t){return i(this,t,arguments.length,arguments[1],!1)}})},function(t,n,r){"use strict";var e=r(1),i=r(135),o=r(45),u=r(75),c=r(16),f=[].slice;e(e.P+e.F*r(4)(function(){i&&f.call(i)}),"Array",{slice:function(t,n){var r=c(this.length),e=o(this);if(n=void 0===n?r:n,"Array"==e)return f.call(this,t,n);for(var i=u(t,r),a=u(n,r),s=c(a-i),l=Array(s),h=0;h<s;h++)l[h]="String"==e?this.charAt(i+h):this[i+h];return l}})},function(t,n,r){"use strict";var e=r(1),i=r(48)(3);e(e.P+e.F*!r(47)([].some,!0),"Array",{some:function(t){return i(this,t,arguments[1])}})},function(t,n,r){"use strict";var e=r(1),i=r(26),o=r(17),u=r(4),c=[].sort,f=[1,2,3];e(e.P+e.F*(u(function(){f.sort(void 0)})||!u(function(){f.sort(null)})||!r(47)(c)),"Array",{sort:function(t){return void 0===t?c.call(o(this)):c.call(o(this),i(t))}})},function(t,n,r){r(74)("Array")},function(t,n,r){var e=r(1);e(e.S,"Date",{now:function(){return(new Date).getTime()}})},function(t,n,r){"use strict";var e=r(1),i=r(4),o=Date.prototype.getTime,u=function(t){return t>9?t:"0"+t};e(e.P+e.F*(i(function(){return"0385-07-25T07:06:39.999Z"!=new Date(-5e13-1).toISOString()})||!i(function(){new Date(NaN).toISOString()})),"Date",{toISOString:function(){
if(!isFinite(o.call(this)))throw RangeError("Invalid time value");var t=this,n=t.getUTCFullYear(),r=t.getUTCMilliseconds(),e=n<0?"-":n>9999?"+":"";return e+("00000"+Math.abs(n)).slice(e?-6:-4)+"-"+u(t.getUTCMonth()+1)+"-"+u(t.getUTCDate())+"T"+u(t.getUTCHours())+":"+u(t.getUTCMinutes())+":"+u(t.getUTCSeconds())+"."+(r>99?r:"0"+u(r))+"Z"}})},function(t,n,r){"use strict";var e=r(1),i=r(17),o=r(50);e(e.P+e.F*r(4)(function(){return null!==new Date(NaN).toJSON()||1!==Date.prototype.toJSON.call({toISOString:function(){return 1}})}),"Date",{toJSON:function(t){var n=i(this),r=o(n);return"number"!=typeof r||isFinite(r)?n.toISOString():null}})},function(t,n,r){var e=r(7)("toPrimitive"),i=Date.prototype;e in i||r(27)(i,e,r(204))},function(t,n,r){var e=Date.prototype,i="Invalid Date",o="toString",u=e[o],c=e.getTime;new Date(NaN)+""!=i&&r(28)(e,o,function(){var t=c.call(this);return t===t?u.call(this):i})},function(t,n,r){var e=r(1);e(e.P,"Function",{bind:r(163)})},function(t,n,r){"use strict";var e=r(6),i=r(32),o=r(7)("hasInstance"),u=Function.prototype;o in u||r(11).f(u,o,{value:function(t){if("function"!=typeof this||!e(t))return!1;if(!e(this.prototype))return t instanceof this;for(;t=i(t);)if(this.prototype===t)return!0;return!1}})},function(t,n,r){var e=r(11).f,i=r(66),o=r(24),u=Function.prototype,c="name",f=Object.isExtensible||function(){return!0};c in u||r(10)&&e(u,c,{configurable:!0,get:function(){try{var t=this,n=(""+t).match(/^\s*function ([^ (]*)/)[1];return o(t,c)||!f(t)||e(t,c,i(5,n)),n}catch(t){return""}}})},function(t,n,r){var e=r(1),i=r(171),o=Math.sqrt,u=Math.acosh;e(e.S+e.F*!(u&&710==Math.floor(u(Number.MAX_VALUE))&&u(1/0)==1/0),"Math",{acosh:function(t){return(t=+t)<1?NaN:t>94906265.62425156?Math.log(t)+Math.LN2:i(t-1+o(t-1)*o(t+1))}})},function(t,n,r){function e(t){return isFinite(t=+t)&&0!=t?t<0?-e(-t):Math.log(t+Math.sqrt(t*t+1)):t}var i=r(1),o=Math.asinh;i(i.S+i.F*!(o&&1/o(0)>0),"Math",{asinh:e})},function(t,n,r){var e=r(1),i=Math.atanh;e(e.S+e.F*!(i&&1/i(-0)<0),"Math",{atanh:function(t){return 0==(t=+t)?t:Math.log((1+t)/(1-t))/2}})},function(t,n,r){var e=r(1),i=r(142);e(e.S,"Math",{cbrt:function(t){return i(t=+t)*Math.pow(Math.abs(t),1/3)}})},function(t,n,r){var e=r(1);e(e.S,"Math",{clz32:function(t){return(t>>>=0)?31-Math.floor(Math.log(t+.5)*Math.LOG2E):32}})},function(t,n,r){var e=r(1),i=Math.exp;e(e.S,"Math",{cosh:function(t){return(i(t=+t)+i(-t))/2}})},function(t,n,r){var e=r(1),i=r(141);e(e.S+e.F*(i!=Math.expm1),"Math",{expm1:i})},function(t,n,r){var e=r(1),i=r(142),o=Math.pow,u=o(2,-52),c=o(2,-23),f=o(2,127)*(2-c),a=o(2,-126),s=function(t){return t+1/u-1/u};e(e.S,"Math",{fround:function(t){var n,r,e=Math.abs(t),o=i(t);return e<a?o*s(e/a/c)*a*c:(n=(1+c/u)*e,r=n-(n-e),r>f||r!=r?o*(1/0):o*r)}})},function(t,n,r){var e=r(1),i=Math.abs;e(e.S,"Math",{hypot:function(t,n){for(var r,e,o=0,u=0,c=arguments.length,f=0;u<c;)r=i(arguments[u++]),f<r?(e=f/r,o=o*e*e+1,f=r):r>0?(e=r/f,o+=e*e):o+=r;return f===1/0?1/0:f*Math.sqrt(o)}})},function(t,n,r){var e=r(1),i=Math.imul;e(e.S+e.F*r(4)(function(){return-5!=i(4294967295,5)||2!=i.length}),"Math",{imul:function(t,n){var r=65535,e=+t,i=+n,o=r&e,u=r&i;return 0|o*u+((r&e>>>16)*u+o*(r&i>>>16)<<16>>>0)}})},function(t,n,r){var e=r(1);e(e.S,"Math",{log10:function(t){return Math.log(t)/Math.LN10}})},function(t,n,r){var e=r(1);e(e.S,"Math",{log1p:r(171)})},function(t,n,r){var e=r(1);e(e.S,"Math",{log2:function(t){return Math.log(t)/Math.LN2}})},function(t,n,r){var e=r(1);e(e.S,"Math",{sign:r(142)})},function(t,n,r){var e=r(1),i=r(141),o=Math.exp;e(e.S+e.F*r(4)(function(){return-2e-17!=!Math.sinh(-2e-17)}),"Math",{sinh:function(t){return Math.abs(t=+t)<1?(i(t)-i(-t))/2:(o(t-1)-o(-t-1))*(Math.E/2)}})},function(t,n,r){var e=r(1),i=r(141),o=Math.exp;e(e.S,"Math",{tanh:function(t){var n=i(t=+t),r=i(-t);return n==1/0?1:r==1/0?-1:(n-r)/(o(t)+o(-t))}})},function(t,n,r){var e=r(1);e(e.S,"Math",{trunc:function(t){return(t>0?Math.floor:Math.ceil)(t)}})},function(t,n,r){"use strict";var e=r(3),i=r(24),o=r(45),u=r(136),c=r(50),f=r(4),a=r(71).f,s=r(31).f,l=r(11).f,h=r(82).trim,v="Number",p=e[v],d=p,y=p.prototype,g=o(r(70)(y))==v,b="trim"in String.prototype,m=function(t){var n=c(t,!1);if("string"==typeof n&&n.length>2){n=b?n.trim():h(n,3);var r,e,i,o=n.charCodeAt(0);if(43===o||45===o){if(88===(r=n.charCodeAt(2))||120===r)return NaN}else if(48===o){switch(n.charCodeAt(1)){case 66:case 98:e=2,i=49;break;case 79:case 111:e=8,i=55;break;default:return+n}for(var u,f=n.slice(2),a=0,s=f.length;a<s;a++)if((u=f.charCodeAt(a))<48||u>i)return NaN;return parseInt(f,e)}}return+n};if(!p(" 0o1")||!p("0b1")||p("+0x1")){p=function(t){var n=arguments.length<1?0:t,r=this;return r instanceof p&&(g?f(function(){y.valueOf.call(r)}):o(r)!=v)?u(new d(m(n)),r,p):m(n)};for(var x,w=r(10)?a(d):"MAX_VALUE,MIN_VALUE,NaN,NEGATIVE_INFINITY,POSITIVE_INFINITY,EPSILON,isFinite,isInteger,isNaN,isSafeInteger,MAX_SAFE_INTEGER,MIN_SAFE_INTEGER,parseFloat,parseInt,isInteger".split(","),S=0;w.length>S;S++)i(d,x=w[S])&&!i(p,x)&&l(p,x,s(d,x));p.prototype=y,y.constructor=p,r(28)(e,v,p)}},function(t,n,r){var e=r(1);e(e.S,"Number",{EPSILON:Math.pow(2,-52)})},function(t,n,r){var e=r(1),i=r(3).isFinite;e(e.S,"Number",{isFinite:function(t){return"number"==typeof t&&i(t)}})},function(t,n,r){var e=r(1);e(e.S,"Number",{isInteger:r(168)})},function(t,n,r){var e=r(1);e(e.S,"Number",{isNaN:function(t){return t!=t}})},function(t,n,r){var e=r(1),i=r(168),o=Math.abs;e(e.S,"Number",{isSafeInteger:function(t){return i(t)&&o(t)<=9007199254740991}})},function(t,n,r){var e=r(1);e(e.S,"Number",{MAX_SAFE_INTEGER:9007199254740991})},function(t,n,r){var e=r(1);e(e.S,"Number",{MIN_SAFE_INTEGER:-9007199254740991})},function(t,n,r){var e=r(1),i=r(178);e(e.S+e.F*(Number.parseFloat!=i),"Number",{parseFloat:i})},function(t,n,r){var e=r(1),i=r(179);e(e.S+e.F*(Number.parseInt!=i),"Number",{parseInt:i})},function(t,n,r){"use strict";var e=r(1),i=r(67),o=r(159),u=r(149),c=1..toFixed,f=Math.floor,a=[0,0,0,0,0,0],s="Number.toFixed: incorrect invocation!",l="0",h=function(t,n){for(var r=-1,e=n;++r<6;)e+=t*a[r],a[r]=e%1e7,e=f(e/1e7)},v=function(t){for(var n=6,r=0;--n>=0;)r+=a[n],a[n]=f(r/t),r=r%t*1e7},p=function(){for(var t=6,n="";--t>=0;)if(""!==n||0===t||0!==a[t]){var r=String(a[t]);n=""===n?r:n+u.call(l,7-r.length)+r}return n},d=function(t,n,r){return 0===n?r:n%2==1?d(t,n-1,r*t):d(t*t,n/2,r)},y=function(t){for(var n=0,r=t;r>=4096;)n+=12,r/=4096;for(;r>=2;)n+=1,r/=2;return n};e(e.P+e.F*(!!c&&("0.000"!==8e-5.toFixed(3)||"1"!==.9.toFixed(0)||"1.25"!==1.255.toFixed(2)||"1000000000000000128"!==(0xde0b6b3a7640080).toFixed(0))||!r(4)(function(){c.call({})})),"Number",{toFixed:function(t){var n,r,e,c,f=o(this,s),a=i(t),g="",b=l;if(a<0||a>20)throw RangeError(s);if(f!=f)return"NaN";if(f<=-1e21||f>=1e21)return String(f);if(f<0&&(g="-",f=-f),f>1e-21)if(n=y(f*d(2,69,1))-69,r=n<0?f*d(2,-n,1):f/d(2,n,1),r*=4503599627370496,(n=52-n)>0){for(h(0,r),e=a;e>=7;)h(1e7,0),e-=7;for(h(d(10,e,1),0),e=n-1;e>=23;)v(1<<23),e-=23;v(1<<e),h(1,1),v(2),b=p()}else h(0,r),h(1<<-n,0),b=p()+u.call(l,a);return a>0?(c=b.length,b=g+(c<=a?"0."+u.call(l,a-c)+b:b.slice(0,c-a)+"."+b.slice(c-a))):b=g+b,b}})},function(t,n,r){"use strict";var e=r(1),i=r(4),o=r(159),u=1..toPrecision;e(e.P+e.F*(i(function(){return"1"!==u.call(1,void 0)})||!i(function(){u.call({})})),"Number",{toPrecision:function(t){var n=o(this,"Number#toPrecision: incorrect invocation!");return void 0===t?u.call(n):u.call(n,t)}})},function(t,n,r){var e=r(1);e(e.S+e.F,"Object",{assign:r(172)})},function(t,n,r){var e=r(1);e(e.S,"Object",{create:r(70)})},function(t,n,r){var e=r(1);e(e.S+e.F*!r(10),"Object",{defineProperties:r(173)})},function(t,n,r){var e=r(1);e(e.S+e.F*!r(10),"Object",{defineProperty:r(11).f})},function(t,n,r){var e=r(6),i=r(65).onFreeze;r(49)("freeze",function(t){return function(n){return t&&e(n)?t(i(n)):n}})},function(t,n,r){var e=r(30),i=r(31).f;r(49)("getOwnPropertyDescriptor",function(){return function(t,n){return i(e(t),n)}})},function(t,n,r){r(49)("getOwnPropertyNames",function(){return r(174).f})},function(t,n,r){var e=r(17),i=r(32);r(49)("getPrototypeOf",function(){return function(t){return i(e(t))}})},function(t,n,r){var e=r(6);r(49)("isExtensible",function(t){return function(n){return!!e(n)&&(!t||t(n))}})},function(t,n,r){var e=r(6);r(49)("isFrozen",function(t){return function(n){return!e(n)||!!t&&t(n)}})},function(t,n,r){var e=r(6);r(49)("isSealed",function(t){return function(n){return!e(n)||!!t&&t(n)}})},function(t,n,r){var e=r(1);e(e.S,"Object",{is:r(180)})},function(t,n,r){var e=r(17),i=r(72);r(49)("keys",function(){return function(t){return i(e(t))}})},function(t,n,r){var e=r(6),i=r(65).onFreeze;r(49)("preventExtensions",function(t){return function(n){return t&&e(n)?t(i(n)):n}})},function(t,n,r){var e=r(6),i=r(65).onFreeze;r(49)("seal",function(t){return function(n){return t&&e(n)?t(i(n)):n}})},function(t,n,r){var e=r(1);e(e.S,"Object",{setPrototypeOf:r(144).set})},function(t,n,r){"use strict";var e=r(114),i={};i[r(7)("toStringTag")]="z",i+""!="[object z]"&&r(28)(Object.prototype,"toString",function(){return"[object "+e(this)+"]"},!0)},function(t,n,r){var e=r(1),i=r(178);e(e.G+e.F*(parseFloat!=i),{parseFloat:i})},function(t,n,r){var e=r(1),i=r(179);e(e.G+e.F*(parseInt!=i),{parseInt:i})},function(t,n,r){"use strict";var e,i,o,u=r(69),c=r(3),f=r(53),a=r(114),s=r(1),l=r(6),h=r(26),v=r(68),p=r(79),d=r(146),y=r(151).set,g=r(143)(),b="Promise",m=c.TypeError,x=c.process,w=c[b],x=c.process,S="process"==a(x),_=function(){},O=!!function(){try{var t=w.resolve(1),n=(t.constructor={})[r(7)("species")]=function(t){t(_,_)};return(S||"function"==typeof PromiseRejectionEvent)&&t.then(_)instanceof n}catch(t){}}(),E=function(t,n){return t===n||t===w&&n===o},P=function(t){var n;return!(!l(t)||"function"!=typeof(n=t.then))&&n},j=function(t){return E(w,t)?new F(t):new i(t)},F=i=function(t){var n,r;this.promise=new t(function(t,e){if(void 0!==n||void 0!==r)throw m("Bad Promise constructor");n=t,r=e}),this.resolve=h(n),this.reject=h(r)},M=function(t){try{t()}catch(t){return{error:t}}},A=function(t,n){if(!t._n){t._n=!0;var r=t._c;g(function(){for(var e=t._v,i=1==t._s,o=0;r.length>o;)!function(n){var r,o,u=i?n.ok:n.fail,c=n.resolve,f=n.reject,a=n.domain;try{u?(i||(2==t._h&&I(t),t._h=1),!0===u?r=e:(a&&a.enter(),r=u(e),a&&a.exit()),r===n.promise?f(m("Promise-chain cycle")):(o=P(r))?o.call(r,c,f):c(r)):f(e)}catch(t){f(t)}}(r[o++]);t._c=[],t._n=!1,n&&!t._h&&N(t)})}},N=function(t){y.call(c,function(){var n,r,e,i=t._v;if(T(t)&&(n=M(function(){S?x.emit("unhandledRejection",i,t):(r=c.onunhandledrejection)?r({promise:t,reason:i}):(e=c.console)&&e.error&&e.error("Unhandled promise rejection",i)}),t._h=S||T(t)?2:1),t._a=void 0,n)throw n.error})},T=function(t){if(1==t._h)return!1;for(var n,r=t._a||t._c,e=0;r.length>e;)if(n=r[e++],n.fail||!T(n.promise))return!1;return!0},I=function(t){y.call(c,function(){var n;S?x.emit("rejectionHandled",t):(n=c.onrejectionhandled)&&n({promise:t,reason:t._v})})},k=function(t){var n=this;n._d||(n._d=!0,n=n._w||n,n._v=t,n._s=2,n._a||(n._a=n._c.slice()),A(n,!0))},L=function(t){var n,r=this;if(!r._d){r._d=!0,r=r._w||r;try{if(r===t)throw m("Promise can't be resolved itself");(n=P(t))?g(function(){var e={_w:r,_d:!1};try{n.call(t,f(L,e,1),f(k,e,1))}catch(t){k.call(e,t)}}):(r._v=t,r._s=1,A(r,!1))}catch(t){k.call({_w:r,_d:!1},t)}}};O||(w=function(t){v(this,w,b,"_h"),h(t),e.call(this);try{t(f(L,this,1),f(k,this,1))}catch(t){k.call(this,t)}},e=function(t){this._c=[],this._a=void 0,this._s=0,this._d=!1,this._v=void 0,this._h=0,this._n=!1},e.prototype=r(73)(w.prototype,{then:function(t,n){var r=j(d(this,w));return r.ok="function"!=typeof t||t,r.fail="function"==typeof n&&n,r.domain=S?x.domain:void 0,this._c.push(r),this._a&&this._a.push(r),this._s&&A(this,!1),r.promise},catch:function(t){return this.then(void 0,t)}}),F=function(){var t=new e;this.promise=t,this.resolve=f(L,t,1),this.reject=f(k,t,1)}),s(s.G+s.W+s.F*!O,{Promise:w}),r(81)(w,b),r(74)(b),o=r(52)[b],s(s.S+s.F*!O,b,{reject:function(t){var n=j(this);return(0,n.reject)(t),n.promise}}),s(s.S+s.F*(u||!O),b,{resolve:function(t){if(t instanceof w&&E(t.constructor,this))return t;var n=j(this);return(0,n.resolve)(t),n.promise}}),s(s.S+s.F*!(O&&r(123)(function(t){w.all(t).catch(_)})),b,{all:function(t){var n=this,r=j(n),e=r.resolve,i=r.reject,o=M(function(){var r=[],o=0,u=1;p(t,!1,function(t){var c=o++,f=!1;r.push(void 0),u++,n.resolve(t).then(function(t){f||(f=!0,r[c]=t,--u||e(r))},i)}),--u||e(r)});return o&&i(o.error),r.promise},race:function(t){var n=this,r=j(n),e=r.reject,i=M(function(){p(t,!1,function(t){n.resolve(t).then(r.resolve,e)})});return i&&e(i.error),r.promise}})},function(t,n,r){var e=r(1),i=r(26),o=r(2),u=(r(3).Reflect||{}).apply,c=Function.apply;e(e.S+e.F*!r(4)(function(){u(function(){})}),"Reflect",{apply:function(t,n,r){var e=i(t),f=o(r);return u?u(e,n,f):c.call(e,n,f)}})},function(t,n,r){var e=r(1),i=r(70),o=r(26),u=r(2),c=r(6),f=r(4),a=r(163),s=(r(3).Reflect||{}).construct,l=f(function(){function t(){}return!(s(function(){},[],t)instanceof t)}),h=!f(function(){s(function(){})});e(e.S+e.F*(l||h),"Reflect",{construct:function(t,n){o(t),u(n);var r=arguments.length<3?t:o(arguments[2]);if(h&&!l)return s(t,n,r);if(t==r){switch(n.length){case 0:return new t;case 1:return new t(n[0]);case 2:return new t(n[0],n[1]);case 3:return new t(n[0],n[1],n[2]);case 4:return new t(n[0],n[1],n[2],n[3])}var e=[null];return e.push.apply(e,n),new(a.apply(t,e))}var f=r.prototype,v=i(c(f)?f:Object.prototype),p=Function.apply.call(t,v,n);return c(p)?p:v}})},function(t,n,r){var e=r(11),i=r(1),o=r(2),u=r(50);i(i.S+i.F*r(4)(function(){Reflect.defineProperty(e.f({},1,{value:1}),1,{value:2})}),"Reflect",{defineProperty:function(t,n,r){o(t),n=u(n,!0),o(r);try{return e.f(t,n,r),!0}catch(t){return!1}}})},function(t,n,r){var e=r(1),i=r(31).f,o=r(2);e(e.S,"Reflect",{deleteProperty:function(t,n){var r=i(o(t),n);return!(r&&!r.configurable)&&delete t[n]}})},function(t,n,r){"use strict";var e=r(1),i=r(2),o=function(t){this._t=i(t),this._i=0;var n,r=this._k=[];for(n in t)r.push(n)};r(139)(o,"Object",function(){var t,n=this,r=n._k;do{if(n._i>=r.length)return{value:void 0,done:!0}}while(!((t=r[n._i++])in n._t));return{value:t,done:!1}}),e(e.S,"Reflect",{enumerate:function(t){return new o(t)}})},function(t,n,r){var e=r(31),i=r(1),o=r(2);i(i.S,"Reflect",{getOwnPropertyDescriptor:function(t,n){return e.f(o(t),n)}})},function(t,n,r){var e=r(1),i=r(32),o=r(2);e(e.S,"Reflect",{getPrototypeOf:function(t){return i(o(t))}})},function(t,n,r){function e(t,n){var r,c,s=arguments.length<3?t:arguments[2];return a(t)===s?t[n]:(r=i.f(t,n))?u(r,"value")?r.value:void 0!==r.get?r.get.call(s):void 0:f(c=o(t))?e(c,n,s):void 0}var i=r(31),o=r(32),u=r(24),c=r(1),f=r(6),a=r(2);c(c.S,"Reflect",{get:e})},function(t,n,r){var e=r(1);e(e.S,"Reflect",{has:function(t,n){return n in t}})},function(t,n,r){var e=r(1),i=r(2),o=Object.isExtensible;e(e.S,"Reflect",{isExtensible:function(t){return i(t),!o||o(t)}})},function(t,n,r){var e=r(1);e(e.S,"Reflect",{ownKeys:r(177)})},function(t,n,r){var e=r(1),i=r(2),o=Object.preventExtensions;e(e.S,"Reflect",{preventExtensions:function(t){i(t);try{return o&&o(t),!0}catch(t){return!1}}})},function(t,n,r){var e=r(1),i=r(144);i&&e(e.S,"Reflect",{setPrototypeOf:function(t,n){i.check(t,n);try{return i.set(t,n),!0}catch(t){return!1}}})},function(t,n,r){function e(t,n,r){var f,h,v=arguments.length<4?t:arguments[3],p=o.f(s(t),n);if(!p){if(l(h=u(t)))return e(h,n,r,v);p=a(0)}return c(p,"value")?!(!1===p.writable||!l(v)||(f=o.f(v,n)||a(0),f.value=r,i.f(v,n,f),0)):void 0!==p.set&&(p.set.call(v,r),!0)}var i=r(11),o=r(31),u=r(32),c=r(24),f=r(1),a=r(66),s=r(2),l=r(6);f(f.S,"Reflect",{set:e})},function(t,n,r){var e=r(3),i=r(136),o=r(11).f,u=r(71).f,c=r(122),f=r(120),a=e.RegExp,s=a,l=a.prototype,h=/a/g,v=/a/g,p=new a(h)!==h;if(r(10)&&(!p||r(4)(function(){return v[r(7)("match")]=!1,a(h)!=h||a(v)==v||"/a/i"!=a(h,"i")}))){a=function(t,n){var r=this instanceof a,e=c(t),o=void 0===n;return!r&&e&&t.constructor===a&&o?t:i(p?new s(e&&!o?t.source:t,n):s((e=t instanceof a)?t.source:t,e&&o?f.call(t):n),r?this:l,a)};for(var d=u(s),y=0;d.length>y;)!function(t){t in a||o(a,t,{configurable:!0,get:function(){return s[t]},set:function(n){s[t]=n}})}(d[y++]);l.constructor=a,a.prototype=l,r(28)(e,"RegExp",a)}r(74)("RegExp")},function(t,n,r){r(119)("match",1,function(t,n,r){return[function(r){"use strict";var e=t(this),i=void 0==r?void 0:r[n];return void 0!==i?i.call(r,e):new RegExp(r)[n](String(e))},r]})},function(t,n,r){r(119)("replace",2,function(t,n,r){return[function(e,i){"use strict";var o=t(this),u=void 0==e?void 0:e[n];return void 0!==u?u.call(e,o,i):r.call(String(o),e,i)},r]})},function(t,n,r){r(119)("search",1,function(t,n,r){return[function(r){"use strict";var e=t(this),i=void 0==r?void 0:r[n];return void 0!==i?i.call(r,e):new RegExp(r)[n](String(e))},r]})},function(t,n,r){r(119)("split",2,function(t,n,e){"use strict";var i=r(122),o=e,u=[].push,c="split",f="length",a="lastIndex";if("c"=="abbc"[c](/(b)*/)[1]||4!="test"[c](/(?:)/,-1)[f]||2!="ab"[c](/(?:ab)*/)[f]||4!="."[c](/(.?)(.?)/)[f]||"."[c](/()()/)[f]>1||""[c](/.?/)[f]){var s=void 0===/()??/.exec("")[1];e=function(t,n){var r=String(this);if(void 0===t&&0===n)return[];if(!i(t))return o.call(r,t,n);var e,c,l,h,v,p=[],d=(t.ignoreCase?"i":"")+(t.multiline?"m":"")+(t.unicode?"u":"")+(t.sticky?"y":""),y=0,g=void 0===n?4294967295:n>>>0,b=new RegExp(t.source,d+"g");for(s||(e=new RegExp("^"+b.source+"$(?!\\s)",d));(c=b.exec(r))&&!((l=c.index+c[0][f])>y&&(p.push(r.slice(y,c.index)),!s&&c[f]>1&&c[0].replace(e,function(){for(v=1;v<arguments[f]-2;v++)void 0===arguments[v]&&(c[v]=void 0)}),c[f]>1&&c.index<r[f]&&u.apply(p,c.slice(1)),h=c[0][f],y=l,p[f]>=g));)b[a]===c.index&&b[a]++;return y===r[f]?!h&&b.test("")||p.push(""):p.push(r.slice(y)),p[f]>g?p.slice(0,g):p}}else"0"[c](void 0,0)[f]&&(e=function(t,n){return void 0===t&&0===n?[]:o.call(this,t,n)});return[function(r,i){var o=t(this),u=void 0==r?void 0:r[n];return void 0!==u?u.call(r,o,i):e.call(String(o),r,i)},e]})},function(t,n,r){"use strict";r(184);var e=r(2),i=r(120),o=r(10),u="toString",c=/./[u],f=function(t){r(28)(RegExp.prototype,u,t,!0)};r(4)(function(){return"/a/b"!=c.call({source:"a",flags:"b"})})?f(function(){var t=e(this);return"/".concat(t.source,"/","flags"in t?t.flags:!o&&t instanceof RegExp?i.call(t):void 0)}):c.name!=u&&f(function(){return c.call(this)})},function(t,n,r){"use strict";r(29)("anchor",function(t){return function(n){return t(this,"a","name",n)}})},function(t,n,r){"use strict";r(29)("big",function(t){return function(){return t(this,"big","","")}})},function(t,n,r){"use strict";r(29)("blink",function(t){return function(){return t(this,"blink","","")}})},function(t,n,r){"use strict";r(29)("bold",function(t){return function(){return t(this,"b","","")}})},function(t,n,r){"use strict";var e=r(1),i=r(147)(!1);e(e.P,"String",{codePointAt:function(t){return i(this,t)}})},function(t,n,r){"use strict";var e=r(1),i=r(16),o=r(148),u="endsWith",c=""[u];e(e.P+e.F*r(134)(u),"String",{endsWith:function(t){var n=o(this,t,u),r=arguments.length>1?arguments[1]:void 0,e=i(n.length),f=void 0===r?e:Math.min(i(r),e),a=String(t);return c?c.call(n,a,f):n.slice(f-a.length,f)===a}})},function(t,n,r){"use strict";r(29)("fixed",function(t){return function(){return t(this,"tt","","")}})},function(t,n,r){"use strict";r(29)("fontcolor",function(t){return function(n){return t(this,"font","color",n)}})},function(t,n,r){"use strict";r(29)("fontsize",function(t){return function(n){return t(this,"font","size",n)}})},function(t,n,r){var e=r(1),i=r(75),o=String.fromCharCode,u=String.fromCodePoint;e(e.S+e.F*(!!u&&1!=u.length),"String",{fromCodePoint:function(t){for(var n,r=[],e=arguments.length,u=0;e>u;){if(n=+arguments[u++],i(n,1114111)!==n)throw RangeError(n+" is not a valid code point");r.push(n<65536?o(n):o(55296+((n-=65536)>>10),n%1024+56320))}return r.join("")}})},function(t,n,r){"use strict";var e=r(1),i=r(148),o="includes";e(e.P+e.F*r(134)(o),"String",{includes:function(t){return!!~i(this,t,o).indexOf(t,arguments.length>1?arguments[1]:void 0)}})},function(t,n,r){"use strict";r(29)("italics",function(t){return function(){return t(this,"i","","")}})},function(t,n,r){"use strict";var e=r(147)(!0);r(140)(String,"String",function(t){this._t=String(t),this._i=0},function(){var t,n=this._t,r=this._i;return r>=n.length?{value:void 0,done:!0}:(t=e(n,r),this._i+=t.length,{value:t,done:!1})})},function(t,n,r){"use strict";r(29)("link",function(t){return function(n){return t(this,"a","href",n)}})},function(t,n,r){var e=r(1),i=r(30),o=r(16);e(e.S,"String",{raw:function(t){for(var n=i(t.raw),r=o(n.length),e=arguments.length,u=[],c=0;r>c;)u.push(String(n[c++])),c<e&&u.push(String(arguments[c]));return u.join("")}})},function(t,n,r){var e=r(1);e(e.P,"String",{repeat:r(149)})},function(t,n,r){"use strict";r(29)("small",function(t){return function(){return t(this,"small","","")}})},function(t,n,r){"use strict";var e=r(1),i=r(16),o=r(148),u="startsWith",c=""[u];e(e.P+e.F*r(134)(u),"String",{startsWith:function(t){var n=o(this,t,u),r=i(Math.min(arguments.length>1?arguments[1]:void 0,n.length)),e=String(t);return c?c.call(n,e,r):n.slice(r,r+e.length)===e}})},function(t,n,r){"use strict";r(29)("strike",function(t){return function(){return t(this,"strike","","")}})},function(t,n,r){"use strict";r(29)("sub",function(t){return function(){return t(this,"sub","","")}})},function(t,n,r){"use strict";r(29)("sup",function(t){return function(){return t(this,"sup","","")}})},function(t,n,r){"use strict";r(82)("trim",function(t){return function(){return t(this,3)}})},function(t,n,r){"use strict";var e=r(3),i=r(24),o=r(10),u=r(1),c=r(28),f=r(65).KEY,a=r(4),s=r(126),l=r(81),h=r(76),v=r(7),p=r(182),d=r(153),y=r(206),g=r(205),b=r(138),m=r(2),x=r(30),w=r(50),S=r(66),_=r(70),O=r(174),E=r(31),P=r(11),j=r(72),F=E.f,M=P.f,A=O.f,N=e.Symbol,T=e.JSON,I=T&&T.stringify,k="prototype",L=v("_hidden"),R=v("toPrimitive"),C={}.propertyIsEnumerable,D=s("symbol-registry"),U=s("symbols"),W=s("op-symbols"),G=Object[k],B="function"==typeof N,V=e.QObject,z=!V||!V[k]||!V[k].findChild,q=o&&a(function(){return 7!=_(M({},"a",{get:function(){return M(this,"a",{value:7}).a}})).a})?function(t,n,r){var e=F(G,n);e&&delete G[n],M(t,n,r),e&&t!==G&&M(G,n,e)}:M,K=function(t){var n=U[t]=_(N[k]);return n._k=t,n},J=B&&"symbol"==typeof N.iterator?function(t){return"symbol"==typeof t}:function(t){return t instanceof N},Y=function(t,n,r){return t===G&&Y(W,n,r),m(t),n=w(n,!0),m(r),i(U,n)?(r.enumerable?(i(t,L)&&t[L][n]&&(t[L][n]=!1),r=_(r,{enumerable:S(0,!1)})):(i(t,L)||M(t,L,S(1,{})),t[L][n]=!0),q(t,n,r)):M(t,n,r)},H=function(t,n){m(t);for(var r,e=g(n=x(n)),i=0,o=e.length;o>i;)Y(t,r=e[i++],n[r]);return t},$=function(t,n){return void 0===n?_(t):H(_(t),n)},X=function(t){var n=C.call(this,t=w(t,!0));return!(this===G&&i(U,t)&&!i(W,t))&&(!(n||!i(this,t)||!i(U,t)||i(this,L)&&this[L][t])||n)},Q=function(t,n){if(t=x(t),n=w(n,!0),t!==G||!i(U,n)||i(W,n)){var r=F(t,n);return!r||!i(U,n)||i(t,L)&&t[L][n]||(r.enumerable=!0),r}},Z=function(t){for(var n,r=A(x(t)),e=[],o=0;r.length>o;)i(U,n=r[o++])||n==L||n==f||e.push(n);return e},tt=function(t){for(var n,r=t===G,e=A(r?W:x(t)),o=[],u=0;e.length>u;)!i(U,n=e[u++])||r&&!i(G,n)||o.push(U[n]);return o};B||(N=function(){if(this instanceof N)throw TypeError("Symbol is not a constructor!");var t=h(arguments.length>0?arguments[0]:void 0),n=function(r){this===G&&n.call(W,r),i(this,L)&&i(this[L],t)&&(this[L][t]=!1),q(this,t,S(1,r))};return o&&z&&q(G,t,{configurable:!0,set:n}),K(t)},c(N[k],"toString",function(){return this._k}),E.f=Q,P.f=Y,r(71).f=O.f=Z,r(116).f=X,r(125).f=tt,o&&!r(69)&&c(G,"propertyIsEnumerable",X,!0),p.f=function(t){return K(v(t))}),u(u.G+u.W+u.F*!B,{Symbol:N});for(var nt="hasInstance,isConcatSpreadable,iterator,match,replace,search,species,split,toPrimitive,toStringTag,unscopables".split(","),rt=0;nt.length>rt;)v(nt[rt++]);for(var nt=j(v.store),rt=0;nt.length>rt;)d(nt[rt++]);u(u.S+u.F*!B,"Symbol",{for:function(t){return i(D,t+="")?D[t]:D[t]=N(t)},keyFor:function(t){if(J(t))return y(D,t);throw TypeError(t+" is not a symbol!")},useSetter:function(){z=!0},useSimple:function(){z=!1}}),u(u.S+u.F*!B,"Object",{create:$,defineProperty:Y,defineProperties:H,getOwnPropertyDescriptor:Q,getOwnPropertyNames:Z,getOwnPropertySymbols:tt}),T&&u(u.S+u.F*(!B||a(function(){var t=N();return"[null]"!=I([t])||"{}"!=I({a:t})||"{}"!=I(Object(t))})),"JSON",{stringify:function(t){if(void 0!==t&&!J(t)){for(var n,r,e=[t],i=1;arguments.length>i;)e.push(arguments[i++]);return n=e[1],"function"==typeof n&&(r=n),!r&&b(n)||(n=function(t,n){if(r&&(n=r.call(this,t,n)),!J(n))return n}),e[1]=n,I.apply(T,e)}}}),N[k][R]||r(27)(N[k],R,N[k].valueOf),l(N,"Symbol"),l(Math,"Math",!0),l(e.JSON,"JSON",!0)},function(t,n,r){"use strict";var e=r(1),i=r(127),o=r(152),u=r(2),c=r(75),f=r(16),a=r(6),s=r(3).ArrayBuffer,l=r(146),h=o.ArrayBuffer,v=o.DataView,p=i.ABV&&s.isView,d=h.prototype.slice,y=i.VIEW,g="ArrayBuffer";e(e.G+e.W+e.F*(s!==h),{ArrayBuffer:h}),e(e.S+e.F*!i.CONSTR,g,{isView:function(t){return p&&p(t)||a(t)&&y in t}}),e(e.P+e.U+e.F*r(4)(function(){return!new h(2).slice(1,void 0).byteLength}),g,{slice:function(t,n){if(void 0!==d&&void 0===n)return d.call(u(this),t);for(var r=u(this).byteLength,e=c(t,r),i=c(void 0===n?r:n,r),o=new(l(this,h))(f(i-e)),a=new v(this),s=new v(o),p=0;e<i;)s.setUint8(p++,a.getUint8(e++));return o}}),r(74)(g)},function(t,n,r){var e=r(1);e(e.G+e.W+e.F*!r(127).ABV,{DataView:r(152).DataView})},function(t,n,r){r(55)("Float32",4,function(t){return function(n,r,e){return t(this,n,r,e)}})},function(t,n,r){r(55)("Float64",8,function(t){return function(n,r,e){return t(this,n,r,e)}})},function(t,n,r){r(55)("Int16",2,function(t){return function(n,r,e){return t(this,n,r,e)}})},function(t,n,r){r(55)("Int32",4,function(t){return function(n,r,e){return t(this,n,r,e)}})},function(t,n,r){r(55)("Int8",1,function(t){return function(n,r,e){return t(this,n,r,e)}})},function(t,n,r){r(55)("Uint16",2,function(t){return function(n,r,e){return t(this,n,r,e)}})},function(t,n,r){r(55)("Uint32",4,function(t){return function(n,r,e){return t(this,n,r,e)}})},function(t,n,r){r(55)("Uint8",1,function(t){return function(n,r,e){return t(this,n,r,e)}})},function(t,n,r){r(55)("Uint8",1,function(t){return function(n,r,e){return t(this,n,r,e)}},!0)},function(t,n,r){"use strict";var e=r(166);r(118)("WeakSet",function(t){return function(){return t(this,arguments.length>0?arguments[0]:void 0)}},{add:function(t){return e.def(this,t,!0)}},e,!1,!0)},function(t,n,r){"use strict";var e=r(1),i=r(117)(!0);e(e.P,"Array",{includes:function(t){return i(this,t,arguments.length>1?arguments[1]:void 0)}}),r(78)("includes")},function(t,n,r){var e=r(1),i=r(143)(),o=r(3).process,u="process"==r(45)(o);e(e.G,{asap:function(t){var n=u&&o.domain;i(n?n.bind(t):t)}})},function(t,n,r){var e=r(1),i=r(45);e(e.S,"Error",{isError:function(t){return"Error"===i(t)}})},function(t,n,r){var e=r(1);e(e.P+e.R,"Map",{toJSON:r(165)("Map")})},function(t,n,r){var e=r(1);e(e.S,"Math",{iaddh:function(t,n,r,e){var i=t>>>0,o=n>>>0,u=r>>>0;return o+(e>>>0)+((i&u|(i|u)&~(i+u>>>0))>>>31)|0}})},function(t,n,r){var e=r(1);e(e.S,"Math",{imulh:function(t,n){var r=65535,e=+t,i=+n,o=e&r,u=i&r,c=e>>16,f=i>>16,a=(c*u>>>0)+(o*u>>>16);return c*f+(a>>16)+((o*f>>>0)+(a&r)>>16)}})},function(t,n,r){var e=r(1);e(e.S,"Math",{isubh:function(t,n,r,e){var i=t>>>0,o=n>>>0,u=r>>>0;return o-(e>>>0)-((~i&u|~(i^u)&i-u>>>0)>>>31)|0}})},function(t,n,r){var e=r(1);e(e.S,"Math",{umulh:function(t,n){var r=65535,e=+t,i=+n,o=e&r,u=i&r,c=e>>>16,f=i>>>16,a=(c*u>>>0)+(o*u>>>16);return c*f+(a>>>16)+((o*f>>>0)+(a&r)>>>16)}})},function(t,n,r){"use strict";var e=r(1),i=r(17),o=r(26),u=r(11);r(10)&&e(e.P+r(124),"Object",{__defineGetter__:function(t,n){u.f(i(this),t,{get:o(n),enumerable:!0,configurable:!0})}})},function(t,n,r){"use strict";var e=r(1),i=r(17),o=r(26),u=r(11);r(10)&&e(e.P+r(124),"Object",{__defineSetter__:function(t,n){u.f(i(this),t,{set:o(n),enumerable:!0,configurable:!0})}})},function(t,n,r){var e=r(1),i=r(176)(!0);e(e.S,"Object",{entries:function(t){return i(t)}})},function(t,n,r){var e=r(1),i=r(177),o=r(30),u=r(31),c=r(131);e(e.S,"Object",{getOwnPropertyDescriptors:function(t){for(var n,r=o(t),e=u.f,f=i(r),a={},s=0;f.length>s;)c(a,n=f[s++],e(r,n));return a}})},function(t,n,r){"use strict";var e=r(1),i=r(17),o=r(50),u=r(32),c=r(31).f;r(10)&&e(e.P+r(124),"Object",{__lookupGetter__:function(t){var n,r=i(this),e=o(t,!0);do{if(n=c(r,e))return n.get}while(r=u(r))}})},function(t,n,r){"use strict";var e=r(1),i=r(17),o=r(50),u=r(32),c=r(31).f;r(10)&&e(e.P+r(124),"Object",{__lookupSetter__:function(t){var n,r=i(this),e=o(t,!0);do{if(n=c(r,e))return n.set}while(r=u(r))}})},function(t,n,r){var e=r(1),i=r(176)(!1);e(e.S,"Object",{values:function(t){return i(t)}})},function(t,n,r){"use strict";var e=r(1),i=r(3),o=r(52),u=r(143)(),c=r(7)("observable"),f=r(26),a=r(2),s=r(68),l=r(73),h=r(27),v=r(79),p=v.RETURN,d=function(t){return null==t?void 0:f(t)},y=function(t){var n=t._c;n&&(t._c=void 0,n())},g=function(t){return void 0===t._o},b=function(t){g(t)||(t._o=void 0,y(t))},m=function(t,n){a(t),this._c=void 0,this._o=t,t=new x(this);try{var r=n(t),e=r;null!=r&&("function"==typeof r.unsubscribe?r=function(){e.unsubscribe()}:f(r),this._c=r)}catch(n){return void t.error(n)}g(this)&&y(this)};m.prototype=l({},{unsubscribe:function(){b(this)}});var x=function(t){this._s=t};x.prototype=l({},{next:function(t){var n=this._s;if(!g(n)){var r=n._o;try{var e=d(r.next);if(e)return e.call(r,t)}catch(t){try{b(n)}finally{throw t}}}},error:function(t){var n=this._s;if(g(n))throw t;var r=n._o;n._o=void 0;try{var e=d(r.error);if(!e)throw t;t=e.call(r,t)}catch(t){try{y(n)}finally{throw t}}return y(n),t},complete:function(t){var n=this._s;if(!g(n)){var r=n._o;n._o=void 0;try{var e=d(r.complete);t=e?e.call(r,t):void 0}catch(t){try{y(n)}finally{throw t}}return y(n),t}}});var w=function(t){s(this,w,"Observable","_f")._f=f(t)};l(w.prototype,{subscribe:function(t){return new m(t,this._f)},forEach:function(t){var n=this;return new(o.Promise||i.Promise)(function(r,e){f(t);var i=n.subscribe({next:function(n){try{return t(n)}catch(t){e(t),i.unsubscribe()}},error:e,complete:r})})}}),l(w,{from:function(t){var n="function"==typeof this?this:w,r=d(a(t)[c]);if(r){var e=a(r.call(t));return e.constructor===n?e:new n(function(t){return e.subscribe(t)})}return new n(function(n){var r=!1;return u(function(){if(!r){try{if(v(t,!1,function(t){if(n.next(t),r)return p})===p)return}catch(t){if(r)throw t;return void n.error(t)}n.complete()}}),function(){r=!0}})},of:function(){for(var t=0,n=arguments.length,r=Array(n);t<n;)r[t]=arguments[t++];return new("function"==typeof this?this:w)(function(t){var n=!1;return u(function(){if(!n){for(var e=0;e<r.length;++e)if(t.next(r[e]),n)return;t.complete()}}),function(){n=!0}})}}),h(w.prototype,c,function(){return this}),e(e.G,{Observable:w}),r(74)("Observable")},function(t,n,r){var e=r(54),i=r(2),o=e.key,u=e.set;e.exp({defineMetadata:function(t,n,r,e){u(t,n,i(r),o(e))}})},function(t,n,r){var e=r(54),i=r(2),o=e.key,u=e.map,c=e.store;e.exp({deleteMetadata:function(t,n){var r=arguments.length<3?void 0:o(arguments[2]),e=u(i(n),r,!1);if(void 0===e||!e.delete(t))return!1;if(e.size)return!0;var f=c.get(n);return f.delete(r),!!f.size||c.delete(n)}})},function(t,n,r){var e=r(185),i=r(161),o=r(54),u=r(2),c=r(32),f=o.keys,a=o.key,s=function(t,n){var r=f(t,n),o=c(t);if(null===o)return r;var u=s(o,n);return u.length?r.length?i(new e(r.concat(u))):u:r};o.exp({getMetadataKeys:function(t){return s(u(t),arguments.length<2?void 0:a(arguments[1]))}})},function(t,n,r){var e=r(54),i=r(2),o=r(32),u=e.has,c=e.get,f=e.key,a=function(t,n,r){if(u(t,n,r))return c(t,n,r);var e=o(n);return null!==e?a(t,e,r):void 0};e.exp({getMetadata:function(t,n){return a(t,i(n),arguments.length<3?void 0:f(arguments[2]))}})},function(t,n,r){var e=r(54),i=r(2),o=e.keys,u=e.key;e.exp({getOwnMetadataKeys:function(t){
return o(i(t),arguments.length<2?void 0:u(arguments[1]))}})},function(t,n,r){var e=r(54),i=r(2),o=e.get,u=e.key;e.exp({getOwnMetadata:function(t,n){return o(t,i(n),arguments.length<3?void 0:u(arguments[2]))}})},function(t,n,r){var e=r(54),i=r(2),o=r(32),u=e.has,c=e.key,f=function(t,n,r){if(u(t,n,r))return!0;var e=o(n);return null!==e&&f(t,e,r)};e.exp({hasMetadata:function(t,n){return f(t,i(n),arguments.length<3?void 0:c(arguments[2]))}})},function(t,n,r){var e=r(54),i=r(2),o=e.has,u=e.key;e.exp({hasOwnMetadata:function(t,n){return o(t,i(n),arguments.length<3?void 0:u(arguments[2]))}})},function(t,n,r){var e=r(54),i=r(2),o=r(26),u=e.key,c=e.set;e.exp({metadata:function(t,n){return function(r,e){c(t,n,(void 0!==e?i:o)(r),u(e))}}})},function(t,n,r){var e=r(1);e(e.P+e.R,"Set",{toJSON:r(165)("Set")})},function(t,n,r){"use strict";var e=r(1),i=r(147)(!0);e(e.P,"String",{at:function(t){return i(this,t)}})},function(t,n,r){"use strict";var e=r(1),i=r(46),o=r(16),u=r(122),c=r(120),f=RegExp.prototype,a=function(t,n){this._r=t,this._s=n};r(139)(a,"RegExp String",function(){var t=this._r.exec(this._s);return{value:t,done:null===t}}),e(e.P,"String",{matchAll:function(t){if(i(this),!u(t))throw TypeError(t+" is not a regexp!");var n=String(this),r="flags"in f?String(t.flags):c.call(t),e=new RegExp(t.source,~r.indexOf("g")?r:"g"+r);return e.lastIndex=o(t.lastIndex),new a(e,n)}})},function(t,n,r){"use strict";var e=r(1),i=r(181);e(e.P,"String",{padEnd:function(t){return i(this,t,arguments.length>1?arguments[1]:void 0,!1)}})},function(t,n,r){"use strict";var e=r(1),i=r(181);e(e.P,"String",{padStart:function(t){return i(this,t,arguments.length>1?arguments[1]:void 0,!0)}})},function(t,n,r){"use strict";r(82)("trimLeft",function(t){return function(){return t(this,1)}},"trimStart")},function(t,n,r){"use strict";r(82)("trimRight",function(t){return function(){return t(this,2)}},"trimEnd")},function(t,n,r){r(153)("asyncIterator")},function(t,n,r){r(153)("observable")},function(t,n,r){var e=r(1);e(e.S,"System",{global:r(3)})},function(t,n,r){for(var e=r(155),i=r(28),o=r(3),u=r(27),c=r(80),f=r(7),a=f("iterator"),s=f("toStringTag"),l=c.Array,h=["NodeList","DOMTokenList","MediaList","StyleSheetList","CSSRuleList"],v=0;v<5;v++){var p,d=h[v],y=o[d],g=y&&y.prototype;if(g){g[a]||u(g,a,l),g[s]||u(g,s,d),c[d]=l;for(p in e)g[p]||i(g,p,e[p],!0)}}},function(t,n,r){var e=r(1),i=r(151);e(e.G+e.B,{setImmediate:i.set,clearImmediate:i.clear})},function(t,n,r){var e=r(3),i=r(1),o=r(121),u=r(207),c=e.navigator,f=!!c&&/MSIE .\./.test(c.userAgent),a=function(t){return f?function(n,r){return t(o(u,[].slice.call(arguments,2),"function"==typeof n?n:Function(n)),r)}:t};i(i.G+i.B+i.F*f,{setTimeout:a(e.setTimeout),setInterval:a(e.setInterval)})},function(t,n,r){r(330),r(269),r(271),r(270),r(273),r(275),r(280),r(274),r(272),r(282),r(281),r(277),r(278),r(276),r(268),r(279),r(283),r(284),r(236),r(238),r(237),r(286),r(285),r(256),r(266),r(267),r(257),r(258),r(259),r(260),r(261),r(262),r(263),r(264),r(265),r(239),r(240),r(241),r(242),r(243),r(244),r(245),r(246),r(247),r(248),r(249),r(250),r(251),r(252),r(253),r(254),r(255),r(317),r(322),r(329),r(320),r(312),r(313),r(318),r(323),r(325),r(308),r(309),r(310),r(311),r(314),r(315),r(316),r(319),r(321),r(324),r(326),r(327),r(328),r(231),r(233),r(232),r(235),r(234),r(220),r(218),r(224),r(221),r(227),r(229),r(217),r(223),r(214),r(228),r(212),r(226),r(225),r(219),r(222),r(211),r(213),r(216),r(215),r(230),r(155),r(302),r(307),r(184),r(303),r(304),r(305),r(306),r(287),r(183),r(185),r(186),r(342),r(331),r(332),r(337),r(340),r(341),r(335),r(338),r(336),r(339),r(333),r(334),r(288),r(289),r(290),r(291),r(292),r(295),r(293),r(294),r(296),r(297),r(298),r(299),r(301),r(300),r(343),r(369),r(372),r(371),r(373),r(374),r(370),r(375),r(376),r(354),r(357),r(353),r(351),r(352),r(355),r(356),r(346),r(368),r(377),r(345),r(347),r(349),r(348),r(350),r(359),r(360),r(362),r(361),r(364),r(363),r(365),r(366),r(367),r(344),r(358),r(380),r(379),r(378),t.exports=r(52)},function(t,n){function r(t,n){if("string"==typeof n)return t.insertAdjacentHTML("afterend",n);var r=t.nextSibling;return r?t.parentNode.insertBefore(n,r):t.parentNode.appendChild(n)}t.exports=r},,,,,,,,,function(t,n,r){(function(n,r){!function(n){"use strict";function e(t,n,r,e){var i=n&&n.prototype instanceof o?n:o,u=Object.create(i.prototype),c=new p(e||[]);return u._invoke=s(t,r,c),u}function i(t,n,r){try{return{type:"normal",arg:t.call(n,r)}}catch(t){return{type:"throw",arg:t}}}function o(){}function u(){}function c(){}function f(t){["next","throw","return"].forEach(function(n){t[n]=function(t){return this._invoke(n,t)}})}function a(t){function n(r,e,o,u){var c=i(t[r],t,e);if("throw"!==c.type){var f=c.arg,a=f.value;return a&&"object"==typeof a&&m.call(a,"__await")?Promise.resolve(a.__await).then(function(t){n("next",t,o,u)},function(t){n("throw",t,o,u)}):Promise.resolve(a).then(function(t){f.value=t,o(f)},u)}u(c.arg)}function e(t,r){function e(){return new Promise(function(e,i){n(t,r,e,i)})}return o=o?o.then(e,e):e()}"object"==typeof r&&r.domain&&(n=r.domain.bind(n));var o;this._invoke=e}function s(t,n,r){var e=P;return function(o,u){if(e===F)throw new Error("Generator is already running");if(e===M){if("throw"===o)throw u;return y()}for(r.method=o,r.arg=u;;){var c=r.delegate;if(c){var f=l(c,r);if(f){if(f===A)continue;return f}}if("next"===r.method)r.sent=r._sent=r.arg;else if("throw"===r.method){if(e===P)throw e=M,r.arg;r.dispatchException(r.arg)}else"return"===r.method&&r.abrupt("return",r.arg);e=F;var a=i(t,n,r);if("normal"===a.type){if(e=r.done?M:j,a.arg===A)continue;return{value:a.arg,done:r.done}}"throw"===a.type&&(e=M,r.method="throw",r.arg=a.arg)}}}function l(t,n){var r=t.iterator[n.method];if(r===g){if(n.delegate=null,"throw"===n.method){if(t.iterator.return&&(n.method="return",n.arg=g,l(t,n),"throw"===n.method))return A;n.method="throw",n.arg=new TypeError("The iterator does not provide a 'throw' method")}return A}var e=i(r,t.iterator,n.arg);if("throw"===e.type)return n.method="throw",n.arg=e.arg,n.delegate=null,A;var o=e.arg;return o?o.done?(n[t.resultName]=o.value,n.next=t.nextLoc,"return"!==n.method&&(n.method="next",n.arg=g),n.delegate=null,A):o:(n.method="throw",n.arg=new TypeError("iterator result is not an object"),n.delegate=null,A)}function h(t){var n={tryLoc:t[0]};1 in t&&(n.catchLoc=t[1]),2 in t&&(n.finallyLoc=t[2],n.afterLoc=t[3]),this.tryEntries.push(n)}function v(t){var n=t.completion||{};n.type="normal",delete n.arg,t.completion=n}function p(t){this.tryEntries=[{tryLoc:"root"}],t.forEach(h,this),this.reset(!0)}function d(t){if(t){var n=t[w];if(n)return n.call(t);if("function"==typeof t.next)return t;if(!isNaN(t.length)){var r=-1,e=function n(){for(;++r<t.length;)if(m.call(t,r))return n.value=t[r],n.done=!1,n;return n.value=g,n.done=!0,n};return e.next=e}}return{next:y}}function y(){return{value:g,done:!0}}var g,b=Object.prototype,m=b.hasOwnProperty,x="function"==typeof Symbol?Symbol:{},w=x.iterator||"@@iterator",S=x.asyncIterator||"@@asyncIterator",_=x.toStringTag||"@@toStringTag",O="object"==typeof t,E=n.regeneratorRuntime;if(E)return void(O&&(t.exports=E));E=n.regeneratorRuntime=O?t.exports:{},E.wrap=e;var P="suspendedStart",j="suspendedYield",F="executing",M="completed",A={},N={};N[w]=function(){return this};var T=Object.getPrototypeOf,I=T&&T(T(d([])));I&&I!==b&&m.call(I,w)&&(N=I);var k=c.prototype=o.prototype=Object.create(N);u.prototype=k.constructor=c,c.constructor=u,c[_]=u.displayName="GeneratorFunction",E.isGeneratorFunction=function(t){var n="function"==typeof t&&t.constructor;return!!n&&(n===u||"GeneratorFunction"===(n.displayName||n.name))},E.mark=function(t){return Object.setPrototypeOf?Object.setPrototypeOf(t,c):(t.__proto__=c,_ in t||(t[_]="GeneratorFunction")),t.prototype=Object.create(k),t},E.awrap=function(t){return{__await:t}},f(a.prototype),a.prototype[S]=function(){return this},E.AsyncIterator=a,E.async=function(t,n,r,i){var o=new a(e(t,n,r,i));return E.isGeneratorFunction(n)?o:o.next().then(function(t){return t.done?t.value:o.next()})},f(k),k[_]="Generator",k.toString=function(){return"[object Generator]"},E.keys=function(t){var n=[];for(var r in t)n.push(r);return n.reverse(),function r(){for(;n.length;){var e=n.pop();if(e in t)return r.value=e,r.done=!1,r}return r.done=!0,r}},E.values=d,p.prototype={constructor:p,reset:function(t){if(this.prev=0,this.next=0,this.sent=this._sent=g,this.done=!1,this.delegate=null,this.method="next",this.arg=g,this.tryEntries.forEach(v),!t)for(var n in this)"t"===n.charAt(0)&&m.call(this,n)&&!isNaN(+n.slice(1))&&(this[n]=g)},stop:function(){this.done=!0;var t=this.tryEntries[0],n=t.completion;if("throw"===n.type)throw n.arg;return this.rval},dispatchException:function(t){function n(n,e){return o.type="throw",o.arg=t,r.next=n,e&&(r.method="next",r.arg=g),!!e}if(this.done)throw t;for(var r=this,e=this.tryEntries.length-1;e>=0;--e){var i=this.tryEntries[e],o=i.completion;if("root"===i.tryLoc)return n("end");if(i.tryLoc<=this.prev){var u=m.call(i,"catchLoc"),c=m.call(i,"finallyLoc");if(u&&c){if(this.prev<i.catchLoc)return n(i.catchLoc,!0);if(this.prev<i.finallyLoc)return n(i.finallyLoc)}else if(u){if(this.prev<i.catchLoc)return n(i.catchLoc,!0)}else{if(!c)throw new Error("try statement without catch or finally");if(this.prev<i.finallyLoc)return n(i.finallyLoc)}}}},abrupt:function(t,n){for(var r=this.tryEntries.length-1;r>=0;--r){var e=this.tryEntries[r];if(e.tryLoc<=this.prev&&m.call(e,"finallyLoc")&&this.prev<e.finallyLoc){var i=e;break}}i&&("break"===t||"continue"===t)&&i.tryLoc<=n&&n<=i.finallyLoc&&(i=null);var o=i?i.completion:{};return o.type=t,o.arg=n,i?(this.method="next",this.next=i.finallyLoc,A):this.complete(o)},complete:function(t,n){if("throw"===t.type)throw t.arg;return"break"===t.type||"continue"===t.type?this.next=t.arg:"return"===t.type?(this.rval=this.arg=t.arg,this.method="return",this.next="end"):"normal"===t.type&&n&&(this.next=n),A},finish:function(t){for(var n=this.tryEntries.length-1;n>=0;--n){var r=this.tryEntries[n];if(r.finallyLoc===t)return this.complete(r.completion,r.afterLoc),v(r),A}},catch:function(t){for(var n=this.tryEntries.length-1;n>=0;--n){var r=this.tryEntries[n];if(r.tryLoc===t){var e=r.completion;if("throw"===e.type){var i=e.arg;v(r)}return i}}throw new Error("illegal catch attempt")},delegateYield:function(t,n,r){return this.delegate={iterator:d(t),resultName:n,nextLoc:r},"next"===this.method&&(this.arg=g),A}}}("object"==typeof n?n:"object"==typeof window?window:"object"==typeof self?self:this)}).call(n,function(){return this}(),r(158))}])</script><script src="/./main.0cf68a.js"></script><script>!function(){!function(e){var t=document.createElement("script");document.getElementsByTagName("body")[0].appendChild(t),t.setAttribute("src",e)}("/slider.e37972.js")}()</script>



    
<div class="tools-col" q-class="show:isShow,hide:isShow|isFalse" q-on="click:stop(e)">
  <div class="tools-nav header-menu">
    
    
      
      
      
    
      
      
      
    
      
      
      
    
    

    <ul style="width: 70%">
    
    
      
      <li style="width: 33.333333333333336%" q-on="click: openSlider(e, 'innerArchive')"><a href="javascript:void(0)" q-class="active:innerArchive">所有文章</a></li>
      
        
      
      <li style="width: 33.333333333333336%" q-on="click: openSlider(e, 'friends')"><a href="javascript:void(0)" q-class="active:friends">友链</a></li>
      
        
      
      <li style="width: 33.333333333333336%" q-on="click: openSlider(e, 'aboutme')"><a href="javascript:void(0)" q-class="active:aboutme">关于我</a></li>
      
        
    </ul>
  </div>
  <div class="tools-wrap">
    
    	<section class="tools-section tools-section-all" q-show="innerArchive">
        <div class="search-wrap">
          <input class="search-ipt" q-model="search" type="text" placeholder="find something…">
          <i class="icon-search icon" q-show="search|isEmptyStr"></i>
          <i class="icon-close icon" q-show="search|isNotEmptyStr" q-on="click:clearChose(e)"></i>
        </div>
        <div class="widget tagcloud search-tag">
          <p class="search-tag-wording">tag:</p>
          <label class="search-switch">
            <input type="checkbox" q-on="click:toggleTag(e)" q-attr="checked:showTags">
          </label>
          <ul class="article-tag-list" q-show="showTags">
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">etcd</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">笔记</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color1">kubernetes</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color2">Statefulset</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color1">k8s组件</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">DaemonSet</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color4">Pod</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color4">HPA</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color1">Deployment</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">RBAC</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color1">k8s安装</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color1">prometheus</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">监控</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color2">Python</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">面向对象</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">野诗</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">古诗</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">随笔</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">hexo</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">性能优化</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">cpu性能优化</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">linux性能优化</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color4">调度器</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">service</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color2">卷</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color4">持久卷</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color2">pv&pvc</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">Configmap</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color2">Secret</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">准入控制</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color4">Qos</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">CronJob</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color5">污点容忍</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color4">亲和力</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color3">linux概念</a>
              </li>
             
              <li class="article-tag-list-item">
                <a href="javascript:void(0)" class="js-tag color1">linux</a>
              </li>
            
            <div class="clearfix"></div>
          </ul>
        </div>
        <ul class="search-ul">
          <p q-show="jsonFail" style="padding: 20px; font-size: 12px;">
            缺失模块。<br/>1、请确保node版本大于6.2<br/>2、在博客根目录（注意不是yilia根目录）执行以下命令：<br/> npm i hexo-generator-json-content --save<br/><br/>
            3、在根目录_config.yml里添加配置：
<pre style="font-size: 12px;" q-show="jsonFail">
  jsonContent:
    meta: false
    pages: false
    posts:
      title: true
      date: true
      path: true
      text: false
      raw: false
      content: false
      slug: false
      updated: false
      comments: false
      link: false
      permalink: false
      excerpt: false
      categories: false
      tags: true
</pre>
          </p>
          <li class="search-li" q-repeat="items" q-show="isShow">
            <a q-attr="href:path|urlformat" class="search-title"><i class="icon-quo-left icon"></i><span q-text="title"></span></a>
            <p class="search-time">
              <i class="icon-calendar icon"></i>
              <span q-text="date|dateformat"></span>
            </p>
            <p class="search-tag">
              <i class="icon-price-tags icon"></i>
              <span q-repeat="tags" q-on="click:choseTag(e, name)" q-text="name|tagformat"></span>
            </p>
          </li>
        </ul>
    	</section>
    

    
    	<section class="tools-section tools-section-friends" q-show="friends">
  		
        <ul class="search-ul">
          
            <li class="search-li">
              <a href="https://blog.csdn.net/CN_SHzhaoyujie" target="_blank" class="search-title"><i class="icon-quo-left icon"></i>CSDN</a>
            </li>
          
        </ul>
  		
    	</section>
    

    
    	<section class="tools-section tools-section-me" q-show="aboutme">
  	  	
  	  		<div class="aboutme-wrap" id="js-aboutme">吾字宸轩，号少翁，凡此皆自娱也。&lt;br&gt;&lt;br&gt;少时不学，尝与玩伴溺于游戏。以至无成，每日叹之，或谓曰：&#34;今天下之势，若有所为，须一技傍身。观Linux厚积薄发当世之所趋，君假以时日，定有所成，成则青云唾手。&#34;&lt;br&gt;吾欣然，心甚向之，随往学。司职至今，四年有余，唯小成，闲无事久，志所不愿也，岂有安居而忘者，恐他日必悔之。故求良木，货与之，望诚以待而寻良遇，志所明而求伯乐。&lt;br&gt;凡shell、lamp、lnmp、jenkins、git、ansible、zabbix、mysql、redis无有不通。python、k8s、docker、prometheus、kvm、kettle无有不会。联系请加微信：542238646</div>
  	  	
    	</section>
    
  </div>
  
</div>
    <!-- Root element of PhotoSwipe. Must have class pswp. -->
<div class="pswp" tabindex="-1" role="dialog" aria-hidden="true">

    <!-- Background of PhotoSwipe. 
         It's a separate element as animating opacity is faster than rgba(). -->
    <div class="pswp__bg"></div>

    <!-- Slides wrapper with overflow:hidden. -->
    <div class="pswp__scroll-wrap">

        <!-- Container that holds slides. 
            PhotoSwipe keeps only 3 of them in the DOM to save memory.
            Don't modify these 3 pswp__item elements, data is added later on. -->
        <div class="pswp__container">
            <div class="pswp__item"></div>
            <div class="pswp__item"></div>
            <div class="pswp__item"></div>
        </div>

        <!-- Default (PhotoSwipeUI_Default) interface on top of sliding area. Can be changed. -->
        <div class="pswp__ui pswp__ui--hidden">

            <div class="pswp__top-bar">

                <!--  Controls are self-explanatory. Order can be changed. -->

                <div class="pswp__counter"></div>

                <button class="pswp__button pswp__button--close" title="Close (Esc)"></button>

                <button class="pswp__button pswp__button--share" style="display:none" title="Share"></button>

                <button class="pswp__button pswp__button--fs" title="Toggle fullscreen"></button>

                <button class="pswp__button pswp__button--zoom" title="Zoom in/out"></button>

                <!-- Preloader demo http://codepen.io/dimsemenov/pen/yyBWoR -->
                <!-- element will get class pswp__preloader--active when preloader is running -->
                <div class="pswp__preloader">
                    <div class="pswp__preloader__icn">
                      <div class="pswp__preloader__cut">
                        <div class="pswp__preloader__donut"></div>
                      </div>
                    </div>
                </div>
            </div>

            <div class="pswp__share-modal pswp__share-modal--hidden pswp__single-tap">
                <div class="pswp__share-tooltip"></div> 
            </div>

            <button class="pswp__button pswp__button--arrow--left" title="Previous (arrow left)">
            </button>

            <button class="pswp__button pswp__button--arrow--right" title="Next (arrow right)">
            </button>

            <div class="pswp__caption">
                <div class="pswp__caption__center"></div>
            </div>

        </div>

    </div>

</div>
  </div>
<script src="/live2dw/lib/L2Dwidget.min.js?094cbace49a39548bed64abff5988b05"></script><script>L2Dwidget.init({"pluginRootPath":"live2dw/","pluginJsPath":"lib/","pluginModelPath":"assets/","tagMode":false,"debug":false,"model":{"jsonPath":"/live2dw/assets/hijiki.model.json"},"display":{"position":"right","width":150,"height":500},"mobile":{"show":true},"log":false});</script></body>
</html>